diff --git a/.editorconfig b/.editorconfig index 25bc5935258bd14..ab1f7ce8425769e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,11 +1,11 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,toml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h,gram}] +[*.{py,c,cpp,h,toml,gram}] indent_size = 4 [*.rst] diff --git a/.gitattributes b/.gitattributes index f4d65dfd1dfbd3b..4d52f900b93179e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -84,6 +84,7 @@ Include/internal/pycore_uop_ids.h generated Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated +Include/slots_generated.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/idlelib/help.html generated @@ -99,7 +100,6 @@ Lib/token.py generated Misc/sbom.spdx.json generated Modules/_testinternalcapi/test_cases.c.h generated Modules/_testinternalcapi/test_targets.h generated -Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated Parser/token.c generated @@ -110,6 +110,7 @@ Python/generated_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/record_functions.c.h generated +Python/slots_generated.c generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index af904a567cfb7e2..0cdf17838b5798f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -63,17 +63,17 @@ .azure-pipelines/ @AA-Turner # GitHub & related scripts -.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz +.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz @itamaro @JacobCoffee Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz Lib/test/test_tools/test_compute_changes.py @AA-Turner @hugovk @webknjaz Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg # Pre-commit -.pre-commit-config.yaml @hugovk -.ruff.toml @hugovk @AlexWaygood @AA-Turner +.pre-commit-config.yaml @hugovk @JacobCoffee +.ruff.toml @hugovk @AlexWaygood @AA-Turner @JacobCoffee # Patchcheck -Tools/patchcheck/ @AA-Turner +Tools/patchcheck/ @AA-Turner @itamaro # ---------------------------------------------------------------------------- @@ -81,11 +81,11 @@ Tools/patchcheck/ @AA-Turner # ---------------------------------------------------------------------------- # Autotools -configure* @erlend-aasland @corona10 @AA-Turner @emmatyping -Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping -Modules/makesetup @erlend-aasland @AA-Turner @emmatyping -Modules/Setup* @erlend-aasland @AA-Turner @emmatyping -Tools/build/regen-configure.sh @AA-Turner +configure* @erlend-aasland @corona10 @AA-Turner @emmatyping @itamaro +Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping @itamaro +Modules/makesetup @erlend-aasland @AA-Turner @emmatyping @itamaro +Modules/Setup* @erlend-aasland @AA-Turner @emmatyping @itamaro +Tools/build/regen-configure.sh @AA-Turner @itamaro # generate-build-details Tools/build/generate-build-details.py @FFY00 @@ -156,7 +156,7 @@ Misc/libabigail.abignore @encukou # ---------------------------------------------------------------------------- # Android -Android/ @mhsmith @freakboy3742 +Platforms/Android/ @mhsmith @freakboy3742 Doc/using/android.rst @mhsmith @freakboy3742 Lib/_android_support.py @mhsmith @freakboy3742 Lib/test/test_android.py @mhsmith @freakboy3742 @@ -164,8 +164,7 @@ Lib/test/test_android.py @mhsmith @freakboy3742 # iOS Doc/using/ios.rst @freakboy3742 Lib/_ios_support.py @freakboy3742 -Apple/ @freakboy3742 -iOS/ @freakboy3742 +Platforms/Apple/ @freakboy3742 # macOS Mac/ @python/macos-team @@ -176,8 +175,8 @@ Lib/test/test__osx_support.py @python/macos-team Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping # WebAssembly (Emscripten) -Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping -Tools/wasm/emscripten @freakboy3742 @emmatyping +Platforms/emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) Platforms/WASI @brettcannon @emmatyping @savannahostrowski @@ -574,9 +573,9 @@ Lib/shutil.py @giampaolo Lib/test/test_shutil.py @giampaolo # Site -Lib/site.py @FFY00 -Lib/test/test_site.py @FFY00 -Doc/library/site.rst @FFY00 +Lib/site.py @FFY00 @warsaw +Lib/test/test_site.py @FFY00 @warsaw +Doc/library/site.rst @FFY00 @warsaw # string.templatelib Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner @@ -587,10 +586,10 @@ Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner **/*sysconfig* @FFY00 # SQLite 3 -Doc/library/sqlite3.rst @berkerpeksag @erlend-aasland -Lib/sqlite3/ @berkerpeksag @erlend-aasland -Lib/test/test_sqlite3/ @berkerpeksag @erlend-aasland -Modules/_sqlite/ @berkerpeksag @erlend-aasland +Doc/library/sqlite3.rst @erlend-aasland +Lib/sqlite3/ @erlend-aasland +Lib/test/test_sqlite3/ @erlend-aasland +Modules/_sqlite/ @erlend-aasland # Subprocess Lib/subprocess.py @gpshead @@ -623,9 +622,6 @@ Modules/_typesmodule.c @AA-Turner Lib/unittest/mock.py @cjw296 Lib/test/test_unittest/testmock/ @cjw296 -# Urllib -**/*robotparser* @berkerpeksag - # Venv **/*venv* @vsajip @FFY00 diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index da70710b7ecfa39..177615621f6b8c7 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -34,13 +34,13 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.15" + - "3.16" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index 470ad581367b103..81ae91e5b0af973 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -27,13 +27,13 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.15" + - "3.16" - "CPython main branch" validations: required: true diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 923720bce0bc3bf..5bf4b40947bea70 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,17 +1,24 @@ # Security Policy -## Supported Versions +Python [provides a security policy and threat model](https://devguide.python.org/security/policy/) +in the Python Developer's Guide documenting what bugs are vulnerabilities, +how to structure reports, and what versions of Python accept reports. -The Python team applies security fixes according to the table -in [the devguide]( -https://devguide.python.org/versions/#supported-versions -). +Python Security Response Team (PSRT) members +balance security work against many other responsibilities. Please be thoughtful +about the time and attention your report requires. Repeated failure to respect +the security policy will result in future reports being rejected, or the +reporter being banned from the ``python`` GitHub organization, regardless of +technical merit. ## Reporting a Vulnerability -Please read the guidelines on reporting security issues [on the -official website](https://www.python.org/dev/security/) for -instructions on how to report a security-related problem to -the Python team responsibly. +The [Python security policy](https://devguide.python.org/security/policy/) +documents [how to submit a vulnerability report](https://devguide.python.org/security/policy/#how-to-submit-a-vulnerability-report) +using GitHub Security Advisories. Please read the security policy +prior to filing a vulnerability report, especially the section on [what information to +include and exclude](https://devguide.python.org/security/policy/#what-to-include-and-how-to-structure-a-vulnerability-report) +in vulnerability reports. Following the security policy means the PSRT can +quickly and efficiently triage your report, not following the security policy +will only delay triaging your report. -To reach the response team, email `security at python dot org`. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1af3a0607f9ad2a..d9a956a6bf53038 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -206,16 +206,16 @@ jobs: strategy: fail-fast: false matrix: - # macos-26 is Apple Silicon, macos-15-intel is Intel. - # macos-15-intel only runs tests against the GIL-enabled CPython. + # macos-26 is Apple Silicon, macos-26-intel is Intel. + # macos-26-intel only runs tests against the GIL-enabled CPython. os: - macos-26 - - macos-15-intel + - macos-26-intel free-threading: - false - true exclude: - - os: macos-15-intel + - os: macos-26-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: @@ -278,14 +278,13 @@ jobs: # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs # supported by important vendors such as AWS-LC. - { name: openssl, version: 1.1.1w } - - { name: openssl, version: 3.0.20 } - - { name: openssl, version: 3.3.7 } - - { name: openssl, version: 3.4.5 } - - { name: openssl, version: 3.5.6 } - - { name: openssl, version: 3.6.2 } - - { name: openssl, version: 4.0.0 } + - { name: openssl, version: 3.0.21 } + - { name: openssl, version: 3.4.6 } + - { name: openssl, version: 3.5.7 } + - { name: openssl, version: 3.6.3 } + - { name: openssl, version: 4.0.1 } ## AWS-LC - - { name: aws-lc, version: 1.72.1 } + - { name: aws-lc, version: 5.0.0 } env: SSLLIB_VER: ${{ matrix.ssllib.version }} MULTISSL_DIR: ${{ github.workspace }}/multissl @@ -399,7 +398,7 @@ jobs: needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.5.6 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -507,7 +506,7 @@ jobs: matrix: os: [ubuntu-24.04] env: - OPENSSL_VER: 3.5.6 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: @@ -602,6 +601,9 @@ jobs: run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" - name: Install cross Python run: make -j8 install + - name: Display build info + run: | + "$BUILD_DIR/cross-python/bin/python3" -m test.pythoninfo - name: Run test subset with host build run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index e63fe9e1284a012..025ff7ecbeeaffa 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -35,6 +35,9 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh - name: Build tier two interpreter run: | ./configure --enable-experimental-jit=interpreter --with-pydebug @@ -61,10 +64,10 @@ jobs: include: - target: i686-pc-windows-msvc/msvc architecture: Win32 - runner: windows-2025-vs2026 + runner: windows-2025 - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-2025-vs2026 + runner: windows-2025 - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-11-arm @@ -152,6 +155,9 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh - name: Build run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} @@ -177,6 +183,9 @@ jobs: - name: JIT without optimizations (Debug) configure_flags: --enable-experimental-jit --with-pydebug test_env: "PYTHON_UOPS_OPTIMIZE=0" + - name: JIT with stress testing (Debug) + configure_flags: --enable-experimental-jit --with-pydebug + test_env: "PYTHON_JIT_STRESS=1" - name: JIT with tail calling interpreter configure_flags: --enable-experimental-jit --with-tail-call-interp --with-pydebug use_clang: true @@ -188,6 +197,9 @@ jobs: - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh - name: Build run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 7f6571ef9545765..d748b6ff63e68a1 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -69,11 +69,11 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: - python-version: "3.13" - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + python-version: "3.15" + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 6201e719ca87de6..f241fccdeb2a32a 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -27,7 +27,7 @@ apt-get -yq --no-install-recommends install \ zlib1g-dev # Workaround missing libmpdec-dev on ubuntu 24.04 by building mpdecimal -# from source. ppa:ondrej/php (launchpad.net) are unreliable +# from source. ppa:ondrej/php (launchpad.net) are unreliable # (https://status.canonical.com) so fetch the tarball directly # from the upstream host. # https://www.bytereef.org/mpdecimal/ diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index 69a780a9aebc25e..38e6dcceb8f47ca 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -63,15 +63,17 @@ jobs: run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" run: python3 Platforms/emscripten make-build-python + - name: "Display build info of the build Python" + run: python3 Platforms/emscripten pythoninfo-build - name: "Make dependencies" run: >- python3 Platforms/emscripten make-dependencies ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }} - - name: "Configure host Python" + - name: "Configure host/Emscripten Python" run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache - - name: "Make host Python" + - name: "Make host/Emscripten Python" run: python3 Platforms/emscripten make-host - - name: "Display build info" - run: python3 Platforms/emscripten run --pythoninfo + - name: "Display build info of the host/Emscripten Python" + run: python3 Platforms/emscripten pythoninfo-host - name: "Test" run: python3 Platforms/emscripten run --test diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index f10503055b2259a..65a7f857fc4c779 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -38,9 +38,8 @@ jobs: run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.5 xz gdbm tcl-tk@9 make - # Because alternate versions are not symlinked into place by default: - brew link --overwrite tcl-tk@9 + brew bundle --file=Misc/Brewfile + brew install make - name: Configure CPython run: | MACOSX_DEPLOYMENT_TARGET=10.15 \ @@ -55,15 +54,15 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.5)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-15-intel' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-26-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 33f6f0ef455fe02..ef36447964cf418 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -82,19 +82,16 @@ jobs: run: make -j4 - name: Display build info run: make pythoninfo - # test_{capi,faulthandler} are skipped under UBSan because - # they raise signals that UBSan with halt_on_error=1 intercepts. - name: Tests run: >- ./python -m test ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} - ${{ inputs.sanitizer == 'UBSan' && '-x test_capi -x test_faulthandler' || '' }} - -j4 + -j4 -W --timeout=900 --slowest - name: Parallel tests if: >- inputs.sanitizer == 'TSan' && fromJSON(inputs.free-threading) - run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 -W --timeout=600 --slowest - name: Display logs if: always() run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index a7e307848af670c..f4321cefa1b5985 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.5.6 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 48fb70cbff80095..4b4888c38444092 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -16,7 +16,6 @@ jobs: timeout-minutes: 60 env: WASMTIME_VERSION: 38.0.3 - CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -54,14 +53,16 @@ jobs: run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" run: python3 Platforms/WASI make-build-python - - name: "Configure host" + - name: "Display build info of the build Python" + run: python3 Platforms/WASI pythoninfo-build + - name: "Configure host/WASI Python" # `--with-pydebug` inferred from configure-build-python run: python3 Platforms/WASI configure-host -- --config-cache env: WASI_SDK_PATH: ${{ steps.install-wasi-sdk.outputs.wasi-sdk-path }} - - name: "Make host" + - name: "Make host/WASI Python" run: python3 Platforms/WASI make-host - - name: "Display build info" - run: make --directory "${CROSS_BUILD_WASI}" pythoninfo + - name: "Display build info of the host/WASI Python" + run: python3 Platforms/WASI pythoninfo-host - name: "Test" run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index a74724323ec15f8..d07b4f7f29e4875 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -17,7 +17,7 @@ env: jobs: build: name: installer for ${{ inputs.arch }} - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 4c8d0c8a2f984fc..dbb192fb8819a4f 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -22,13 +22,11 @@ permissions: env: FORCE_COLOR: 1 - IncludeUwp: >- - true jobs: build: name: Build and test (${{ inputs.arch }}, ${{ inputs.interpreter }}) - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 01fe5ba8fda8bcb..37f78519dedc329 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -21,10 +21,10 @@ jobs: uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' + stale-pr-message: 'This PR is stale because it has been open for 90 days with no activity.' stale-pr-label: 'stale' days-before-issue-stale: -1 - days-before-pr-stale: 30 + days-before-pr-stale: 90 days-before-close: -1 ascending: true operations-per-run: 120 diff --git a/.gitignore b/.gitignore index 660a2524144d21a..118eb5ee76e8051 100644 --- a/.gitignore +++ b/.gitignore @@ -141,6 +141,7 @@ Tools/unicode/data/ /cross-build*/ /jit_stencils*.h /jit_unwind_info*.h +.jit-stamp /platform /profile-clean-stamp /profile-run-stamp diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6878a7d92e3beeb..6c7f1b93e66e879 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: e05c5c0818279e5ac248ac9e954431ba58865e61 # frozen: v0.15.7 + rev: 3b3f7c3f57fe9925356faf5fe6230835138be230 # frozen: v0.15.17 hooks: - id: ruff-check name: Run Ruff (lint) on Platforms/Apple/ @@ -10,6 +10,11 @@ repos: name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ + - id: ruff-check + name: Run Ruff (lint) on Lib/ + args: [--exit-non-zero-on-fix] + files: ^Lib/ + exclude: ^Lib/test/ - id: ruff-check name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] @@ -18,6 +23,11 @@ repos: name: Run Ruff (lint) on Platforms/WASI/ args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] files: ^Platforms/WASI/ + - id: ruff-check + name: Run Ruff (lint) on Tools/ + args: [--exit-non-zero-on-fix] + files: ^Tools/ + exclude: ^Tools/(build|clinic|i18n|peg_generator|wasm)/ - id: ruff-check name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] diff --git a/.readthedocs.yml b/.readthedocs.yml index 0a2c3f8345367f4..038417e4bb34385 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -11,24 +11,50 @@ build: os: ubuntu-24.04 tools: python: "3" + apt_packages: + - jq - commands: - # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition - # - # Cancel building pull requests when there aren't changes in the Doc directory. - # - # If there are no changes (git diff exits with 0) we force the command to return with 183. - # This is a special exit code on Read the Docs that will cancel the build immediately. - - | - if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ]; - then - echo "No changes to Doc/ - exiting the build."; - exit 183; - fi - - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - - make -C Doc venv html - - mkdir _readthedocs - - mv Doc/build/html _readthedocs/html + jobs: + post_system_dependencies: + # https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html#skip-builds-based-on-conditions + # + # Cancel building pull requests when there are no changes in the Doc + # directory or RTD configuration, or if we can't cleanly merge the base + # branch. + - | + set -eEux; + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; + then + base_branch=$(wget -qO- "https://api.github.com/repos/python/cpython/pulls/$READTHEDOCS_VERSION" | jq -er ".base.ref"); + git fetch --depth=50 origin $base_branch:origin-$base_branch; + for attempt in $(seq 10); + do + if ! git merge-base HEAD origin-$base_branch; + then + git fetch --deepen=50 origin $base_branch; + else + break; + fi; + done; + if ! git -c "user.name=rtd" -c "user.email=no-reply@readthedocs.org" merge --no-stat --no-edit origin-$base_branch; + then + echo "Unsuccessful merge with '$base_branch' branch, skipping the build"; + exit 183; + fi; + if git diff --exit-code --stat origin-$base_branch -- Doc/ .readthedocs.yml; + then + echo "No changes to Doc/ - skipping the build."; + exit 183; + fi; + fi; + create_environment: + - echo "Skipping default environment creation" + install: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - make -C Doc venv html + - mkdir -p "$READTHEDOCS_OUTPUT" + - mv Doc/build/html "$READTHEDOCS_OUTPUT/" diff --git a/Doc/Makefile b/Doc/Makefile index 60970d50833f147..d77ece1e681bcfc 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -88,9 +88,9 @@ htmlhelp: build "build/htmlhelp/pydoc.hhp project file." .PHONY: latex -latex: _ensure-sphinxcontrib-svg2pdfconverter latex: BUILDER = latex -latex: build +latex: _ensure-sphinxcontrib-svg2pdfconverter + $(MAKE) build BUILDER=$(BUILDER) @echo "Build finished; the LaTeX files are in build/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 254a22f2622bd8e..a6ea0a72e76f9db 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -12,6 +12,9 @@ It can be sometimes faster to fix bugs yourself and contribute patches to Python as it streamlines the process and involves fewer people. Learn how to :ref:`contribute `. + +.. _reporting-documentation-bugs: + Documentation bugs ================== diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index f56bcd6333a37d3..fa77d3d38ff89fd 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -384,14 +384,18 @@ Getters Get the writer size. + The function does not invalidate pointers returned by + :c:func:`PyBytesWriter_GetData`. + The function cannot fail. .. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) Get the writer data: start of the internal buffer. - The pointer is valid until :c:func:`PyBytesWriter_Finish` or - :c:func:`PyBytesWriter_Discard` is called on *writer*. + The pointer remains valid until a :c:type:`PyBytesWriter` function other + than :c:func:`PyBytesWriter_GetData` or :c:func:`PyBytesWriter_GetSize` is + called on *writer*. The function cannot fail. diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 629312bd771beb2..10f96c7cb75e882 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -130,7 +130,7 @@ rather than dereferencing them through pointers. Please note, that these functions are :term:`soft deprecated` since Python 3.15. Avoid using this API in a new code to do complex arithmetic: either use -the `Number Protocol `_ API or use native complex types, like +the :ref:`Number Protocol ` API or use native complex types, like :c:expr:`double complex`. diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index caa855a10d20ca9..06422fb63f34f1a 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -33,3 +33,9 @@ return. with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index a2a0d0d80657ebf..556113a97bf772f 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -151,7 +151,7 @@ Dictionary objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. The first argument can be a :class:`dict` or a :class:`frozendict`. diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 7a07818b7b4d1a1..82f594e11300a7d 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -412,7 +412,7 @@ an error value). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - Function similar to :c:func:`PyErr_WarnEx`, but use + Function similar to :c:func:`PyErr_WarnEx`, but uses :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is an ASCII-encoded string. @@ -1038,7 +1038,7 @@ Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requ special recursion handling. In addition to protecting the stack, :c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. +these are the C equivalent to :deco:`reprlib.recursive_repr`. .. c:function:: int Py_ReprEnter(PyObject *object) @@ -1348,3 +1348,67 @@ Tracebacks This function returns ``0`` on success, and returns ``-1`` with an exception set on failure. + +.. c:function:: const char* PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) + + Write a trace of the Python stack in *tstate* into the file *fd*. The format + looks like:: + + Traceback (most recent call first): + File "xxx", line xxx in + File "xxx", line xxx in + ... + File "xxx", line xxx in + + This function is meant to debug situations such as segfaults, fatal errors, + and similar. The file and function names it outputs are encoded to ASCII with + backslashreplace and truncated to 500 characters. It writes only the first + 100 frames; further frames are truncated with the line ``...``. + + This function will return ``NULL`` on success, or an error message on error. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *tstate* need to be attached. + + .. versionadded:: 3.15 + +.. c:function:: const char* PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState *current_tstate, Py_ssize_t max_threads) + + Write the traces of all Python threads in *interp* into the file *fd*. + + If *interp* is ``NULL`` then this function will try to identify the current + interpreter using thread-specific storage. If it cannot, it will return an + error. + + If *current_tstate* is not ``NULL`` then it will be used to identify what the + current thread is in the written output. If it is ``NULL`` then this function + will identify the current thread using thread-specific storage. It is not an + error if the function is unable to get the current Python thread state. + + This function will return ``NULL`` on success, or an error message on error. + + This function is meant to debug situations such as segfaults, fatal + errors, and similar. It calls :c:func:`PyUnstable_DumpTraceback` for each + thread. It only writes the tracebacks of the first *max_threads* threads, + further output is truncated with the line ``...``. If *max_threads* is 0, the + function will use a default value of 100 for the argument. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *current_tstate* need to be attached. + + .. warning:: + On the :term:`free-threaded build`, this function is not thread-safe. If + another thread deletes its :term:`thread state` while this function is being + called, the process will likely crash. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 7bc04970b19503c..879b1ec50369ff4 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -38,7 +38,7 @@ Extension export hook The export hook must be an exported function with the following signature: -.. c:function:: PyModuleDef_Slot *PyModExport_modulename(void) +.. c:function:: PySlot *PyModExport_modulename(void) For modules with ASCII-only names, the :ref:`export hook ` must be named :samp:`PyModExport_{}`, @@ -57,7 +57,7 @@ Python's *punycode* encoding with hyphens replaced by underscores. In Python: suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') return b'PyModExport' + suffix -The export hook returns an array of :c:type:`PyModuleDef_Slot` entries, +The export hook returns an array of :c:type:`PySlot` entries, terminated by an entry with a slot ID of ``0``. These slots describe how the module should be created and initialized. @@ -75,7 +75,7 @@ It is recommended to define the export hook function using a helper macro: Declare an extension module export hook. This macro: - * specifies the :c:expr:`PyModuleDef_Slot*` return type, + * specifies the :c:expr:`PySlot*` return type, * adds any special linkage declarations required by the platform, and * for C++, declares the function as ``extern "C"``. @@ -83,12 +83,12 @@ For example, a module called ``spam`` would be defined like this:: PyABIInfo_VAR(abi_info); - static PyModuleDef_Slot spam_slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "spam"}, - {Py_mod_init, spam_init_function}, + static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_FUNC(Py_mod_init, spam_init_function), ... - {0, NULL}, + PySlot_END }; PyMODEXPORT_FUNC @@ -100,11 +100,35 @@ For example, a module called ``spam`` would be defined like this:: The export hook is typically the only non-\ ``static`` item defined in the module's C source. -The hook should be kept short -- ideally, one line as above. -If you do need to use Python C API in this function, it is recommended to call -``PyABIInfo_Check(&abi_info, "modulename")`` first to raise an exception, -rather than crash, in common cases of ABI mismatch. +.. _pymodexport-api-caveats: +The hook should be kept short. +If it does more than ``return`` a static array, several caveats apply: + +- If you need to use any Python C API, it is recommended to call + :c:func:`PyABIInfo_Check` first to raise an exception, + rather than crash, in common cases of ABI mismatch. +- Code in the export hook must never rely on the :term:`GIL`: + :term:`free-threaded builds ` of Python can only check + the :c:macro:`Py_mod_gil` slot (or the lack of it) after the hook returns, +- Similarly, the hook may be called in any subinterpreter, since the + :c:macro:`Py_mod_multiple_interpreters` slot (or lack of it) + is only checked after the hook returns. + +For example:: + + PyMODEXPORT_FUNC + PyModExport_modulename(void) + { + if (PyABIInfo_Check(&abi_info, "modulename") < 0) { + /* ABI mismatch. It's not safe to examine the raised exception. */ + return NULL; + } + + /* use Python API (as little as possible); don't rely on GIL */ + + return modulename_slots; + } .. note:: diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 9c9c97f7b853d85..9c71bb3d59d5e99 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -414,7 +414,7 @@ In some cases, failure is not distinguishable from a successful ``NULL`` result. See :ref:`duringgc-functions` for common information. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: @@ -433,7 +433,7 @@ In some cases, failure is not distinguishable from a successful ``NULL`` result. The reference is valid for the duration of the :c:member:`!tp_traverse` handler call. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: :c:func:`PyType_GetBaseByToken` @@ -445,7 +445,7 @@ In some cases, failure is not distinguishable from a successful ``NULL`` result. These functions return a :term:`borrowed reference`, which is valid for the duration of the :c:member:`!tp_traverse` handler call. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index ed121726b89620c..7713ba2ee4f8047 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -38,6 +38,12 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. A reference to *frame* is stolen by this function. The argument must not be ``NULL``. + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + .. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) Create and return a new generator object based on the *frame* object, @@ -45,6 +51,12 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + .. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen) @@ -77,6 +89,12 @@ Asynchronous Generator Objects .. versionadded:: 3.6 + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + .. c:function:: int PyAsyncGen_CheckExact(PyObject *op) Return true if *op* is an asynchronous generator object, false otherwise. diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index e2d363b911a87c6..b48cf951137e511 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -304,6 +304,11 @@ Importing Modules Initialization function for a module built into the interpreter. + Note that the inittab uses "``PyInit``" + :ref:`initialization functions `; + there is currently no way to include "``PyModExport_``" + :ref:`export hooks `. + .. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) @@ -393,11 +398,6 @@ Importing Modules Make all imports lazy by default. - .. c:enumerator:: PyImport_LAZY_NONE - - Disable lazy imports entirely. Even explicit ``lazy`` statements become - eager imports. - .. versionadded:: 3.15 .. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index eabe00f4004001f..051f6fd765e8506 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -18,6 +18,7 @@ document the API functions in detail. refcounting.rst exceptions.rst extension-modules.rst + slots.rst utilities.rst abstract.rst concrete.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 209e48767ccfd6c..d6b9837987a3999 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -623,6 +623,10 @@ Some options are read from the :mod:`sys` attributes. For example, the option .. versionadded:: 3.14 + .. versionchanged:: next + The function now replaces :data:`sys.flags` (create a new object), + instead of modifying :data:`sys.flags` in-place. + .. _pyconfig_api: diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index 186ab4370bcb9c1..46f5b1dd33963c0 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -104,7 +104,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_DebugFlag @@ -119,7 +119,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_DontWriteBytecodeFlag @@ -134,7 +134,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_FrozenFlag @@ -145,7 +145,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_HashRandomizationFlag @@ -161,7 +161,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_IgnoreEnvironmentFlag @@ -175,7 +175,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-E` and :option:`-I` options. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_InspectFlag @@ -191,7 +191,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_InteractiveFlag @@ -202,7 +202,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_IsolatedFlag @@ -218,7 +218,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_LegacyWindowsFSEncodingFlag @@ -238,7 +238,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_LegacyWindowsStdioFlag @@ -257,7 +257,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_NoSiteFlag @@ -273,7 +273,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_NoUserSiteDirectory @@ -288,7 +288,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_OptimizeFlag @@ -300,7 +300,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_QuietFlag @@ -315,7 +315,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_UnbufferedStdioFlag @@ -329,7 +329,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_VerboseFlag @@ -346,7 +346,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 Initializing and finalizing the interpreter @@ -410,7 +410,7 @@ Initializing and finalizing the interpreter (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until :c:func:`Py_Initialize` is called again. - .. versionchanged:: next + .. versionchanged:: 3.15 This function no longer returns true until initialization has fully completed, including import of the :mod:`site` module. Previously it could return true while :c:func:`Py_Initialize` was still running. @@ -578,31 +578,203 @@ Initializing and finalizing the interpreter .. _cautions-regarding-runtime-finalization: -Cautions regarding runtime finalization ---------------------------------------- +Cautions regarding interpreter finalization +------------------------------------------- In the late stage of :term:`interpreter shutdown`, after attempting to wait for non-daemon threads to exit (though this can be interrupted by :class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`Py_IsFinalizing` and -:func:`sys.is_finalizing` return true. At this point, only the *finalization -thread* that initiated finalization (typically the main thread) is allowed to -acquire the :term:`GIL`. - -If any thread, other than the finalization thread, attempts to attach a :term:`thread state` -during finalization, either explicitly or -implicitly, the thread enters **a permanently blocked state** -where it remains until the program exits. In most cases this is harmless, but this can result -in deadlock if a later stage of finalization attempts to acquire a lock owned by the -blocked thread, or otherwise waits on the blocked thread. - -Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ -finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs -have never had any error reporting or handling expectations at :term:`thread state` -attachment time that would've allowed for graceful exit from this situation. Changing that -would require new stable C APIs and rewriting the majority of C code in the -CPython ecosystem to use those with error handling. +is marked as finalizing, meaning that :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the finalization +thread (the thread that initiated finalization; this is typically the main thread) +is allowed to :term:`attach ` a thread state. + +Other threads that attempt to attach during finalization, either explicitly +(such as via :c:func:`PyThreadState_Ensure` or :c:macro:`Py_END_ALLOW_THREADS`) +or implicitly (such as in-between bytecode instructions), will enter a +**permanently blocked state**. Generally, this is harmless, but this can +result in deadlocks. For example, a thread may be permanently blocked while +holding a lock, meaning that the finalization thread can never acquire that +lock. + +Prior to CPython 3.13, the thread would exit instead of hanging, +which led to other issues (see the warning note at +:c:func:`PyThread_exit_thread`). + +Gross? Yes. Starting in Python 3.15, there are a number of C APIs that make +it possible to avoid these issues by temporarily preventing finalization: + +.. _interpreter-guards: + +.. seealso:: + + :pep:`788` explains the design, motivation and rationale + for these APIs. + +.. c:type:: PyInterpreterGuard + + An opaque interpreter guard structure. + + By holding an interpreter guard, the caller can ensure that the interpreter + will not finalize until the guard is closed (through + :c:func:`PyInterpreterGuard_Close`). + + When a guard is held, a thread attempting to finalize the interpreter will + block until the guard is closed before starting finalization. + After finalization has started, threads are forever unable to acquire + guards for that interpreter. This means that if you forget to close an + interpreter guard, the process will **permanently hang** during + finalization! + + Holding a guard for an interpreter is similar to holding a + :term:`strong reference` to a Python object, except finalization does not happen + automatically after all guards are released: it requires an explicit + :c:func:`Py_EndInterpreter` call. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromCurrent(void) + + Create a finalization guard for the current interpreter. This will prevent + finalization until the guard is closed. + + For example: + + .. code-block:: c + + // Temporarily prevent finalization. + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + if (guard == NULL) { + // Finalization has already started or we're out of memory. + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + // Do some critical processing here. For example, we can safely acquire + // locks that might be acquired by the finalization thread. + Py_END_ALLOW_THREADS; + + // Now that we're done with our critical processing, the interpreter is + // allowed to finalize again. + PyInterpreterGuard_Close(guard); + + On success, this function returns a guard for the current interpreter; + on failure, it returns ``NULL`` with an exception set. + + This function will fail only if the current interpreter has already started + finalizing, or if the process is out of memory. + + The guard pointer returned by this function must be eventually closed + with :c:func:`PyInterpreterGuard_Close`; failing to do so will result in + the Python process infinitely hanging. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromView(PyInterpreterView *view) + + Create a finalization guard for an interpreter through a view. + + On success, this function returns a guard to the interpreter + represented by *view*. The view is still valid after calling this + function. The guard must eventually be closed with + :c:func:`PyInterpreterGuard_Close`. + + If the interpreter no longer exists, is already finalizing, or out of memory, + then this function returns ``NULL`` without setting an exception. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: void PyInterpreterGuard_Close(PyInterpreterGuard *guard) + + Close an interpreter guard, allowing the interpreter to start + finalization if no other guards remain. If an interpreter guard + is never closed, the interpreter will infinitely wait when trying + to enter finalization! + + After an interpreter guard is closed, it may not be used in + :c:func:`PyThreadState_Ensure`. Doing so will result in undefined + behavior. + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. _interpreter-views: + +Interpreter views +----------------- + +In some cases, it may be necessary to access an interpreter that may have been +deleted. This can be done using interpreter views. + +.. c:type:: PyInterpreterView + + An opaque view of an interpreter. + + This is a thread-safe way to access an interpreter that may have be + finalizing or already destroyed. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromCurrent(void) + + Create a view to the current interpreter. + + This function is generally meant to be used alongside + :c:func:`PyInterpreterGuard_FromView` or :c:func:`PyThreadState_EnsureFromView`. + + On success, this function returns a view to the current interpreter; on + failure, it returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: void PyInterpreterView_Close(PyInterpreterView *view) + + Close an interpreter view. + + If an interpreter view is never closed, the view's memory will never be + freed, but there are no other consequences. (In contrast, forgetting to + close a guard will infinitely hang the main thread during finalization.) + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromMain(void) + + Create a view for the main interpreter (the first and default + interpreter in a Python process; see + :c:func:`PyInterpreterState_Main`). + + On success, this function returns a view to the main + interpreter; on failure, it returns ``NULL`` without an exception set. + Failure indicates that the process is out of memory. + + Use this function when an interpreter pointer or view cannot be supplied + by the caller, such as when a native threading library does not provide a + ``void *arg`` parameter that could carry a :c:type:`PyInterpreterGuard` or + :c:type:`PyInterpreterView`. In code that supports subinterpreters, prefer + :c:func:`PyInterpreterView_FromCurrent` so the guard tracks the calling + interpreter rather than the main one. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.15 Process-wide parameters @@ -632,7 +804,7 @@ Process-wide parameters Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a :c:expr:`wchar_t*` string. - .. deprecated-removed:: 3.11 3.15 + .. deprecated-removed:: 3.11 3.16 .. c:function:: const char* Py_GetVersion() @@ -757,7 +929,7 @@ Process-wide parameters .. versionadded:: 3.1.3 - .. deprecated-removed:: 3.11 3.15 + .. deprecated-removed:: 3.11 3.16 .. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) @@ -778,7 +950,7 @@ Process-wide parameters .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - .. deprecated-removed:: 3.11 3.15 + .. deprecated-removed:: 3.11 3.16 .. c:function:: void Py_SetPythonHome(const wchar_t *home) @@ -799,4 +971,4 @@ Process-wide parameters Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a :c:expr:`wchar_t*` string. - .. deprecated-removed:: 3.11 3.15 + .. deprecated-removed:: 3.11 3.16 diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 60e3ae4a064e729..874e422d4701dd8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -71,6 +71,12 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) + + Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, + or ``NULL`` on failure. + + .. c:function:: PyObject* PyLong_FromInt32(int32_t value) PyObject* PyLong_FromInt64(int64_t value) @@ -81,12 +87,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.14 -.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, - or ``NULL`` on failure. - - .. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) PyObject* PyLong_FromUInt64(uint64_t value) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 9f84e4bc6dfd91d..73310670ac371c9 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -77,7 +77,7 @@ memory footprint as a whole. Consequently, under certain circumstances, the Python memory manager may or may not trigger appropriate actions, like garbage collection, memory compaction or other preventive procedures. Note that by using the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. +for the I/O buffer completely escapes the Python memory manager. .. seealso:: @@ -157,7 +157,7 @@ zero bytes. .. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -235,7 +235,7 @@ In the GIL-enabled build (default build) the .. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -368,7 +368,7 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index b67ca671a2a1185..9f68abba66bc5d6 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -133,14 +133,16 @@ Module Objects unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. +.. _c_module_slots: .. _pymoduledef_slot: Module definition ----------------- Modules created using the C API are typically defined using an -array of :dfn:`slots`. -The slots provide a "description" of how a module should be created. +array of :c:type:`PySlot` structs, which provides a "description" of how a +module should be created. +See :ref:`capi-slots` for more information on slots in general. .. versionchanged:: 3.15 @@ -158,30 +160,12 @@ Unless specified otherwise, the same slot ID may not be repeated in an array of slots. -.. c:type:: PyModuleDef_Slot - - .. c:member:: int slot - - A slot ID, chosen from the available ``Py_mod_*`` values explained below. - - An ID of 0 marks the end of a :c:type:`!PyModuleDef_Slot` array. - - .. c:member:: void* value - - Value of the slot, whose meaning depends on the slot ID. - - The value may not be NULL. - To leave a slot out, omit the :c:type:`PyModuleDef_Slot` entry entirely. - - .. versionadded:: 3.5 - - Metadata slots .............. .. c:macro:: Py_mod_name - :c:type:`Slot ID ` for the name of the new module, + :c:member:`Slot ID ` for the name of the new module, as a NUL-terminated UTF8-encoded ``const char *``. Note that modules are typically created using a @@ -196,7 +180,7 @@ Metadata slots .. c:macro:: Py_mod_doc - :c:type:`Slot ID ` for the docstring of the new + :c:type:`Slot ID ` for the docstring of the new module, as a NUL-terminated UTF8-encoded ``const char *``. Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`. @@ -211,7 +195,7 @@ Feature slots .. c:macro:: Py_mod_abi - :c:type:`Slot ID ` whose value points to + :c:member:`Slot ID ` whose value points to a :c:struct:`PyABIInfo` structure describing the ABI that the extension is using. @@ -222,8 +206,8 @@ Feature slots PyABIInfo_VAR(abi_info); - static PyModuleDef_Slot mymodule_slots[] = { - {Py_mod_abi, &abi_info}, + static PySlot mymodule_slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), ... }; @@ -237,7 +221,7 @@ Feature slots .. c:macro:: Py_mod_multiple_interpreters - :c:type:`Slot ID ` whose value is one of: + :c:member:`Slot ID ` whose value is one of: .. c:namespace:: NULL @@ -263,11 +247,20 @@ Feature slots If ``Py_mod_multiple_interpreters`` is not specified, the import machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. + For historical reasons, the values are declared as pointers (``void *``). + When using :c:type:`PySlot` arrays, use :c:macro:`PySlot_DATA` for + :c:macro:`!Py_mod_multiple_interpreters`: + + .. code-block:: c + + PySlot_DATA(Py_mod_multiple_interpreters, + Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) + .. versionadded:: 3.12 .. c:macro:: Py_mod_gil - :c:type:`Slot ID ` whose value is one of: + :c:member:`Slot ID ` whose value is one of: .. c:namespace:: NULL @@ -288,6 +281,14 @@ Feature slots If ``Py_mod_gil`` is not specified, the import machinery defaults to ``Py_MOD_GIL_USED``. + For historical reasons, the values are declared as pointers (``void *``). + When using :c:type:`PySlot` arrays, use :c:macro:`PySlot_DATA` for + :c:macro:`!Py_mod_gil`: + + .. code-block:: c + + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED) + .. versionadded:: 3.13 @@ -296,7 +297,7 @@ Creation and initialization slots .. c:macro:: Py_mod_create - :c:type:`Slot ID ` for a function that creates + :c:member:`Slot ID ` for a function that creates the module object itself. The function must have the signature: @@ -346,7 +347,7 @@ Creation and initialization slots .. c:macro:: Py_mod_exec - :c:type:`Slot ID ` for a function that will + :c:member:`Slot ID ` for a function that will :dfn:`execute`, or initialize, the module. This function does the equivalent to executing the code of a Python module: typically, it adds classes and constants to the module. @@ -375,7 +376,7 @@ Creation and initialization slots .. c:macro:: Py_mod_methods - :c:type:`Slot ID ` for a table of module-level + :c:member:`Slot ID ` for a table of module-level functions, as an array of :c:type:`PyMethodDef` values suitable as the *functions* argument to :c:func:`PyModule_AddFunctions`. @@ -446,12 +447,12 @@ To retrieve the state from a given module, use the following functions: Slots for defining module state ............................... -The following :c:member:`PyModuleDef_Slot.slot` IDs are available for +The following :c:member:`slot IDs ` are available for defining the module state. .. c:macro:: Py_mod_state_size - :c:type:`Slot ID ` for the size of the module state, + :c:member:`Slot ID ` for the size of the module state, in bytes. Setting the value to a non-negative value means that the module can be @@ -468,7 +469,7 @@ defining the module state. .. c:macro:: Py_mod_state_traverse - :c:type:`Slot ID ` for a traversal function to call + :c:member:`Slot ID ` for a traversal function to call during GC traversal of the module object. The signature of the function, and meanings of the arguments, @@ -491,7 +492,7 @@ defining the module state. .. c:macro:: Py_mod_state_clear - :c:type:`Slot ID ` for a clear function to call + :c:member:`Slot ID ` for a clear function to call during GC clearing of the module object. The signature of the function is: @@ -519,7 +520,7 @@ defining the module state. .. c:macro:: Py_mod_state_free - :c:type:`Slot ID ` for a function to call during + :c:member:`Slot ID ` for a function to call during deallocation of the module object. The signature of the function is: @@ -578,7 +579,7 @@ A module's token -- and the *your_token* value to use in the above code -- is: .. c:macro:: Py_mod_token - :c:type:`Slot ID ` for the module token. + :c:member:`Slot ID ` for the module token. If you use this slot to set the module token (rather than rely on the default), you must ensure that: @@ -617,14 +618,14 @@ Creating extension modules dynamically The following functions may be used to create an extension module dynamically, rather than from an extension's :ref:`export hook `. -.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) +.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec) Create a new module object, given an array of :ref:`slots ` and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. - The *slots* argument must point to an array of :c:type:`PyModuleDef_Slot` + The *slots* argument must point to an array of :c:type:`PySlot` structures, terminated by an entry with slot ID of 0 - (typically written as ``{0}`` or ``{0, NULL}`` in C). + (typically written as :c:macro:`PySlot_END`). The array must include a :c:data:`Py_mod_abi` entry. The *spec* argument may be any ``ModuleSpec``-like object, as described @@ -640,10 +641,6 @@ rather than from an extension's :ref:`export hook `. must be called to fully initialize a module. (See also :ref:`multi-phase-initialization`.) - The *slots* array only needs to be valid for the duration of the - :c:func:`!PyModule_FromSlotsAndSpec` call. - In particular, it may be heap-allocated. - .. versionadded:: 3.15 .. c:function:: int PyModule_Exec(PyObject *module) @@ -738,6 +735,8 @@ remove it. .. c:member:: PyModuleDef_Slot* m_slots An array of additional slots, terminated by a ``{0, NULL}`` entry. + Note that the entries use the older :c:type:`PyModuleDef_Slot` structure, + rather than :c:type:`PySlot`. If the array contains slots corresponding to :c:type:`PyModuleDef` members, the values must match. @@ -752,6 +751,39 @@ remove it. .. c:member:: inquiry m_reload + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Slot + + Older structure defining additional slots of a module. + + Note that a :c:type:`!PyModuleDef_Slot` array may be included in a + :c:type:`!PySlot` array using :c:macro:`Py_mod_slots`, + and vice versa using :c:macro:`Py_slot_subslots`. + + Each :c:type:`!PyModuleDef_Slot` structure ``modslot`` is interpreted + as the following :c:type:`PySlot` structure:: + + (PySlot){ + .sl_id=modslot.slot, + .sl_flags=PySlot_INTPTR | sub_static, + .sl_ptr=modslot.value + } + + where ``sub_static`` is ``PySlot_STATIC`` if the slot requires + the flag (such as for :c:macro:`Py_mod_methods`), or if this flag + is present on the "parent" :c:macro:`!Py_mod_slots` slot (if any). + + .. c:member:: int slot + + Corresponds to :c:member:`PySlot.sl_id`. + + .. c:member:: void* value + + Corresponds to :c:member:`PySlot.sl_ptr`. + + .. versionadded:: 3.5 + .. c:member:: traverseproc m_traverse inquiry m_clear freefunc m_free @@ -774,6 +806,14 @@ remove it. The type of ``PyModuleDef`` objects. +.. c:macro:: Py_mod_slots + + :c:member:`Slot ID ` that works like + :c:macro:`Py_slot_subslots`, except it specifies an array of + :c:type:`PyModuleDef_Slot` structures. + + .. versionadded:: 3.15 + .. _moduledef-dynamic: The following API can be used to create modules from a :c:type:`!PyModuleDef` @@ -813,6 +853,10 @@ struct: .. versionadded:: 3.5 + .. soft-deprecated:: next + + Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. + .. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) Create a new module object, given the definition in *def* and the @@ -833,12 +877,22 @@ struct: .. versionadded:: 3.5 + .. soft-deprecated:: next + + Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. + .. c:function:: int PyModule_ExecDef(PyObject *module, PyModuleDef *def) Process any execution slots (:c:data:`Py_mod_exec`) given in *def*. .. versionadded:: 3.5 + .. soft-deprecated:: next + + To run a module's own execution slots, prefer :c:func:`PyModule_Exec`, + which works on modules that were not created from a + :c:type:`PyModuleDef` structure. + .. c:macro:: PYTHON_API_VERSION PYTHON_API_STRING diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index bd05e628faaaa10..a962c4ee09ad77d 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -49,3 +49,43 @@ Note that holding an :term:`attached thread state` is not required for these API This is called by the runtime itself during interpreter shut-down. In general, there shouldn't be a reason to explicitly call this, except to handle specific scenarios such as forking. + +.. c:function:: int PyUnstable_CopyPerfMapFile(const char *parent_filename) + + Open the ``/tmp/perf-$pid.map`` file and append the content of *parent_filename* + to it. + + This function is available on all platforms but only generates output on platforms + that support perf maps (currently only Linux). On other platforms, it does nothing. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *code) + + Compile the given code object using the current perf trampoline. + + The "current" trampoline is the one set by the runtime or the most recent + :c:func:`PyUnstable_PerfTrampoline_SetPersistAfterFork` call. + + If no trampoline is set, falls back to normal compilation (no perf map entry). + + :param code: The code object to compile. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable) + + Set whether the perf trampoline should persist after a fork. + + * If ``enable`` is true (non-zero): perf map file remains open/valid post-fork. + Child process inherits all existing perf map entries. + * If ``enable`` is false (zero): perf map closes post-fork. + Child process gets empty perf map. + + Default: false (clears on fork). + + :param enable: 1 to enable, 0 to disable. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst index 710ded56e2a6db4..b1b7329a5d42c59 100644 --- a/Doc/c-api/sentinel.rst +++ b/Doc/c-api/sentinel.rst @@ -10,21 +10,33 @@ Sentinel objects This instance of :c:type:`PyTypeObject` represents the Python :class:`sentinel` type. This is the same object as :class:`sentinel`. - .. versionadded:: next + .. versionadded:: 3.15 .. c:function:: int PySentinel_Check(PyObject *o) - Return true if *o* is a :class:`sentinel` object. The :class:`sentinel` type - does not allow subclasses, so this check is exact. + Return true if *o* is a :class:`sentinel` object or a subtype. + The :class:`sentinel` type does not currently allow subclasses, + so this check is exact. + Future Python versions may choose to allow subtyping. + This function always succeeds. - .. versionadded:: next + .. versionadded:: 3.15 -.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name) +.. c:function:: int PySentinel_CheckExact(PyObject *o) + + Return true if *o* is a :class:`sentinel` object, but not a subtype. + The :class:`sentinel` type does not currently allow subclasses. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name, const char *repr) Return a new :class:`sentinel` object with :attr:`~sentinel.__name__` set to *name* and :attr:`~sentinel.__module__` set to *module_name*. *name* must not be ``NULL``. If *module_name* is ``NULL``, :attr:`~sentinel.__module__` - is set to ``None``. + is set to ``None``. If *repr* is ``NULL``, ``repr()`` returns :attr:`~sentinel.__name__`. Return ``NULL`` with an exception set on failure. For pickling to work, *module_name* must be the name of an importable @@ -32,4 +44,4 @@ Sentinel objects path matching *name*. Pickle treats *name* as a global variable name in *module_name* (see :meth:`object.__reduce__`). - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/c-api/slots.rst b/Doc/c-api/slots.rst new file mode 100644 index 000000000000000..84a125cb60bae7e --- /dev/null +++ b/Doc/c-api/slots.rst @@ -0,0 +1,240 @@ +.. highlight:: c + +.. _capi-slots: + +Definition slots +================ + +To define :ref:`module objects ` and +:ref:`classes ` using the C API, you may use +an array of *slots* -- essentially, key-value pairs that describe features +of the object to create. +This decouples the data from the structures used at runtime, allowing CPython +-- and other Python C API implementations -- to update the structures without +breaking backwards compatibility. + +This section documents slots in general. +For object-specific behavior and slot values, see documentation for functions +that apply slots: + +- :c:func:`PyType_FromSlots` for types; +- :c:func:`PyModule_FromSlotsAndSpec` and :ref:`extension-export-hook` + for modules. + +When slots are passed to a function that applies them, the function will +not modify the slot array, nor any data it points to (recursively). +After the function is done, the caller is allowed to modify or deallocate +the array and any data it points to (recursively), except data +explicitly marked with :c:macro:`PySlot_STATIC`. + +Except when documented otherwise, multiple slots with the same ID +(:c:member:`~PySlot.sl_id`) may not occur in a single slots array. + +.. versionadded:: 3.15 + + Slot arrays generalize an earlier way of defining objects: + using :c:type:`PyType_Spec` with :c:type:`PyType_Slot` for types, and + :c:type:`PyModuleDef` with :c:type:`PyModuleDef_Slot` for modules. + The earlier API is :term:`soft deprecated`; there are no plans to remove it. + +Entries of the slots array use the following structure: + +.. c:type:: PySlot + + An entry in a slots array. Defined as: + + .. code-block:: c + + typedef struct { + uint16_t sl_id; + uint16_t sl_flags; + uint32_t _reserved; // must be 0 + union { + void *sl_ptr; + void (*sl_func)(void); + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; + } PySlot; + + .. c:member:: uint16_t sl_id + + A slot ID, chosen from: + + - ``Py_slot_*`` values documented in :ref:`pyslot-common-ids` below; + - ``Py_mod_*`` values for modules, as documented in :ref:`c_module_slots`; + - Values for types, as documented in :ref:`pyslot_type_slot_ids`. + + A :c:member:`!sl_id` of zero (:c:macro:`Py_slot_end`) marks the end of a + slots array. + + .. c:member:: void *sl_ptr + void (*sl_func)(void) + Py_ssize_t sl_size + int64_t sl_int64 + uint64_t sl_uint64 + + The data for the slot. + These members are part of an anonymous union; + the member to use depends on which data type is required by the slot ID: + data pointer, function pointer, size, signed or unsigned + integer, respectively. + + Except when documented otherwise for a specific slot ID, pointers + (that is :c:member:`!sl_ptr` and :c:member:`!sl_func`) may not be NULL. + + .. c:member:: uint16_t sl_flags + + Zero or more of the following flags, OR-ed together: + + .. c:namespace:: NULL + + .. c:macro:: PySlot_STATIC + + All data the slot points to is statically allocated and constant. + Thus, the interpreter does not need to copy the information. + + This flag is implied for function pointers. + + The flag applies even to data the slot points to “indirectly”, + except for slots nested via :c:macro:`Py_slot_subslots` which may + have their own :c:macro:`!PySlot_STATIC` flags. + For example, if applied to a :c:macro:`Py_tp_members` slot that + points to an array of :c:type:`PyMemberDef` structures, + then the entire array, as well as the name and doc strings + in its elements, must be static and constant. + + .. c:macro:: PySlot_INTPTR + + The data is stored in ``sl_ptr``; CPython will cast it to + the appropriate type. + + This flag can simplify porting from the older :c:type:`PyType_Slot` + and :c:type:`PyModuleDef_Slot` structures. + + .. c:macro:: PySlot_OPTIONAL + + If the slot ID is unknown, the interpreter should ignore the + slot, rather than fail. + + For example, if Python 3.16 adds a new feature with a new slot ID,attr + the corresponding slot may be marked :c:macro:`!PySlot_OPTIONAL` + so that Python 3.15 ignores it. + + Note that the "optionality" only applies to unknown slot IDs. + This flag does not make Python skip invalid values of known slots. + + .. versionadded:: 3.15 + + +Convenience macros +------------------ + +.. c:macro:: PySlot_DATA(name, value) + PySlot_FUNC(name, value) + PySlot_SIZE(name, value) + PySlot_INT64(name, value) + PySlot_UINT64(name, value) + PySlot_STATIC_DATA(name, value) + + Convenience macros to define :c:type:`!PySlot` structures with + :c:member:`~PySlot.sl_id` and a particular union member set. + + :c:macro:`!PySlot_STATIC_DATA` sets the :c:macro:`PySlot_STATIC` flag; + others set no flags. + + Note that these macros use *designated initializers*, a C language feature + that C++ added in the 2020 version of the standard. + If your code needs to be compatible with C++11 or older, + use :c:macro:`PySlot_PTR` instead. + + Defined as:: + + #define PySlot_DATA(NAME, VALUE) \ + {.sl_id=NAME, .sl_ptr=(void*)(VALUE)} + + #define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=NAME, .sl_func=(VALUE)} + + #define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=NAME, .sl_size=(VALUE)} + + #define PySlot_INT64(NAME, VALUE) \ + {.sl_id=NAME, .sl_int64=(VALUE)} + + #define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=NAME, .sl_uint64=(VALUE)} + + #define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=NAME, .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + + .. versionadded:: 3.15 + +.. c:macro:: PySlot_END + + Convenience macro to mark the end of a :c:type:`!PySlot` array. + + Defined as:: + + #define PySlot_END {0} + + .. versionadded:: 3.15 + +.. c:macro:: PySlot_PTR(name, value) + PySlot_PTR_STATIC(name, value) + + Convenience macros for use in C++11-compatible code. + This version of C++ does not allow setting arbitrary union members in + literals; instead, these macros set the :c:macro:`PySlot_INTPTR` flag and cast + the value to ``(void*)``. + + Defined as:: + + #define PySlot_PTR(NAME, VALUE) \ + {NAME, PySlot_INTPTR, {0}, {(void*)(VALUE)}} + + #define PySlot_PTR_STATIC(NAME, VALUE) \ + {NAME, PySlot_INTPTR|Py_SLOT_STATIC, {0}, {(void*)(VALUE)}} + + .. versionadded:: 3.15 + +.. _pyslot-common-ids: + +Common slot IDs +--------------- + +The following slot IDs may be used in both type and module definitions. + +.. c:macro:: Py_slot_end + + Marks the end of a slots array. + Defined as zero. + + .. versionadded:: 3.15 + +.. c:macro:: Py_slot_subslots + + Nested slots array. + + The value (:c:member:`~PySlot.sl_ptr`) should point to an array of + :c:type:`PySlot` structures. + The slots in the array (up to but not including the zero-ID + terminator) will be treated as if they were inserted if the current + slot array, at the point :c:macro:`!Py_slot_subslots` appears. + + Slot nesting depth is limited to 5 levels. + This restriction may be lifted in the future. + + .. versionadded:: 3.15 + +.. c:macro:: Py_slot_invalid + + Reserved; will always be treated as an unknown slot ID. + Defined as ``UINT16_MAX`` (``0xFFFF``). + + When used with the :c:macro:`PySlot_OPTIONAL` flag, defines a slot with + no effect. + Without the flag, processing a slot with this ID will fail. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index fe92f72f8ebd01a..13e5d5c96135c0e 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -74,7 +74,7 @@ There are two Stable ABIs: :term:`free-threaded ` builds of CPython. It has stricter API limitations than ``abi3``. - .. versionadded:: next + .. versionadded:: 3.15 ``abi3t`` was added in :pep:`803` @@ -114,7 +114,7 @@ versions of Python. All functions in Stable ABI are present as functions in Python's shared library, not solely as macros. -This makes them usable are usable from languages that don't use the C +This makes them usable in languages that don't use the C preprocessor, including Python's :py:mod:`ctypes`. @@ -170,7 +170,7 @@ both Stable ABIs. :term:`free-threaded builds ` of CPython. See :ref:`above ` for common information. - .. versionadded:: next + .. versionadded:: 3.15 Both macros specify a target ABI; the different naming style is due to backwards compatibility. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index aeca412610317fe..1ddde5d136b3bb1 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -454,8 +454,8 @@ method. The method will be passed the type object as the first parameter rather than an instance of the type. This is used to create *class methods*, - similar to what is created when using the :func:`classmethod` built-in - function. + similar to what is created when using the :deco:`classmethod` built-in + decorator. .. c:macro:: METH_STATIC @@ -464,7 +464,7 @@ method. The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to - what is created when using the :func:`staticmethod` built-in function. + what is created when using the :deco:`staticmethod` built-in decorator. One other constant controls whether a method is loaded in place of another definition with the same method name. diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst index 53c9faeae35464d..6f18c047a24a92f 100644 --- a/Doc/c-api/synchronization.rst +++ b/Doc/c-api/synchronization.rst @@ -84,11 +84,6 @@ there is no :c:type:`PyObject` -- for example, when working with a C type that does not extend or wrap :c:type:`PyObject` but still needs to call into the C API in a manner that might lead to deadlocks. -The functions and structs used by the macros are exposed for cases -where C macros are not available. They should only be used as in the -given macro expansions. Note that the sizes and contents of the structures may -change in future Python versions. - .. note:: Operations that need to lock two objects at once must use @@ -114,12 +109,15 @@ section API avoids potential deadlocks due to reentrancy and lock ordering by allowing the runtime to temporarily suspend the critical section if the code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. +.. _critical-section-macros: + .. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) Acquires the per-object lock for the object *op* and begins a critical section. - In the free-threaded build, this macro expands to:: + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: { PyCriticalSection _py_cs; @@ -150,7 +148,8 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. Ends the critical section and releases the per-object lock. - In the free-threaded build, this macro expands to:: + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: PyCriticalSection_End(&_py_cs); } @@ -179,7 +178,8 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. Locks the mutexes *m1* and *m2* and begins a critical section. - In the free-threaded build, this macro expands to:: + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: { PyCriticalSection2 _py_cs2; @@ -196,7 +196,8 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. Ends the critical section and releases the per-object locks. - In the free-threaded build, this macro expands to:: + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: PyCriticalSection2_End(&_py_cs2); } @@ -205,6 +206,48 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +Low-level critical section API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions and structs are exposed for cases where C macros +are not available. + +.. c:function:: void PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) + void PyCriticalSection_End(PyCriticalSection *c) + void PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) + void PyCriticalSection2_End(PyCriticalSection2 *c); + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + + In non-:term:`free-threaded ` builds of CPython, these + functions do nothing. + + .. versionadded:: 3.13 + +.. c:type:: PyCriticalSection + PyCriticalSection2 + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + Note that the contents of the structures are private and their meaning may + change in future Python versions. + + .. versionadded:: 3.13 + +.. c:function:: void PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); + void PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); + + .. (These need to be in a separate section without a Stable ABI annotation.) + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + + In non-:term:`free-threaded ` builds of CPython, these + functions do nothing. + + .. versionadded:: 3.14 + Legacy locking APIs ------------------- diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index 3b761d0c657cbd3..ca34abd73d8423b 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -61,9 +61,9 @@ as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. -If it turns out that you do need to create a thread state, call :c:func:`PyThreadState_New` -followed by :c:func:`PyThreadState_Swap`, or use the dangerous -:c:func:`PyGILState_Ensure` function. +If it turns out that you do need to create a thread state, it is recommended to +use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, +which will manage the thread state for you. .. _detaching-thread-state: @@ -178,8 +178,12 @@ example usage in the Python source distribution. declaration. -Non-Python created threads --------------------------- +.. _non-python-created-threads: +.. _c-api-foreign-threads: + + +Using the C API from foreign threads +------------------------------------ When threads are created using the dedicated Python APIs (such as the :mod:`threading` module), a thread state is automatically associated with them, @@ -192,70 +196,275 @@ of a callback API provided by the aforementioned third-party library), you must first register these threads with the interpreter by creating a new thread state and attaching it. -The most robust way to do this is through :c:func:`PyThreadState_New` followed -by :c:func:`PyThreadState_Swap`. +The easiest way to do this is through :c:func:`PyThreadState_Ensure` +or :c:func:`PyThreadState_EnsureFromView`. .. note:: - ``PyThreadState_New`` requires an argument pointing to the desired + These functions require an argument pointing to the desired interpreter; such a pointer can be acquired via a call to - :c:func:`PyInterpreterState_Get` from the code where the thread was - created. + :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or + :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) + from the function that creates the thread. If no pointer is available (such + as when the given native thread library doesn't provide a data argument), + :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main + interpreter, but note that this will make the code incompatible with + subinterpreters. -For example:: - /* The return value of PyInterpreterState_Get() from the - function that created this thread. */ - PyInterpreterState *interp = thread_data->interp; +For example:: - /* Create a new thread state for the interpreter. It does not start out - attached. */ - PyThreadState *tstate = PyThreadState_New(interp); + // The return value of PyInterpreterGuard_FromCurrent() from the + // function that created this thread. + PyInterpreterGuard *guard = thread_data->guard; - /* Attach the thread state, which will acquire the GIL. */ - PyThreadState_Swap(tstate); + // Create a new thread state for the interpreter. + PyThreadStateToken *token = PyThreadState_Ensure(guard); + if (token == NULL) { + PyInterpreterGuard_Close(guard); + return; + } - /* Perform Python actions here. */ + // We have a valid thread state -- perform Python actions here. result = CallSomeFunction(); - /* evaluate result or handle exception */ + // Evaluate result or handle exceptions. - /* Destroy the thread state. No Python API allowed beyond this point. */ - PyThreadState_Clear(tstate); - PyThreadState_DeleteCurrent(); + // Release the thread state. No calls to the C API are allowed beyond this + // point. + PyThreadState_Release(token); + PyInterpreterGuard_Close(guard); -.. warning:: - If the interpreter finalized before ``PyThreadState_Swap`` was called, then - ``interp`` will be a dangling pointer! +Keep in mind that calling ``PyThreadState_Ensure`` might not always create a new +thread state, and calling ``PyThreadState_Release`` might not always detach it. +These functions may reuse an existing attached thread state, or may re-attach +a thread state that was previously attached for the current thread. + +.. seealso:: + :pep:`788` + +.. _c-api-attach-detach: + +Attaching/detaching thread states +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyThreadStateToken *PyThreadState_Ensure(PyInterpreterGuard *guard) + + Ensure that the thread has an attached thread state for the + interpreter protected by *guard*, and thus can safely invoke that + interpreter. + + It is OK to call this function if the thread already has an + attached thread state, as long as there is a subsequent call to + :c:func:`PyThreadState_Release` that matches this one (meaning that "nested" + calls to this function are permitted). + + The function's effect (if any) will be reversed by the matching call to + :c:func:`PyThreadState_Release`. + + On error, this function returns ``NULL`` *without* an exception set. + Do not call :c:func:`!PyThreadState_Release` in this case. + + On success, this function returns a pointer value that must be passed + to the matching call to :c:func:`!PyThreadState_Release`. + + The conditions in which this function creates a new :term:`thread state` are + considered unstable and implementation-dependent. If you need to control the + exact lifetime of a thread state, consider using :c:func:`PyThreadState_New`. + However, do not avoid this function solely on the basis that the lifetime + of the thread state may be inconsistent across versions; changes to this + function will be done with caution and in a backwards-compatible manner. + In particular, the saving of thread-local variables and similar state will + be retained across Python versions. + + .. impl-detail:: + + The exact behavior of whether this function creates a new thread state is + described below, but be aware that this may change in the future. + + First, this function checks if an attached thread state is present. + If there is, this function then checks if the interpreter of that + thread state matches the interpreter guarded by *guard*. If that is + the case, this function simply marks the thread state as being used + by a ``PyThreadState_Ensure`` call and returns. + + If there is no attached thread state, then this function checks if any + thread state has been used by the current OS thread. (This is + returned by :c:func:`PyGILState_GetThisThreadState`.) + If there was, then this function checks if that thread state's interpreter + matches *guard*. If it does, it is re-attached and marked as used. + + Otherwise, if both of the above cases fail, a new thread state is created + for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. + + .. versionadded:: 3.15 + + +.. c:function:: PyThreadStateToken *PyThreadState_EnsureFromView(PyInterpreterView *view) + + Get an attached thread state for the interpreter referenced by *view*. + + The behavior and return value are the same as for :c:func:`PyThreadState_Ensure`; + additionally, if the function succeeds, the interpreter referenced by *view* will + be implicitly guarded. The guard will be released upon the corresponding + :c:func:`PyThreadState_Release` call. + + .. versionadded:: 3.15 + + +.. c:function:: void PyThreadState_Release(PyThreadStateToken *token) + + Undo a :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` call. + + This must be called exactly once for each successful *Ensure* call, with + *token* set to that call's return value. + + The state that was attached before the corresponding *Ensure* call + (if any) will be attached when :c:func:`PyThreadState_Release` returns. + + The exact behavior of whether this function deletes a thread state is + considered unstable and implementation-dependent. + .. impl-detail:: + + Currently, this function will decrement an internal counter on the + attached thread state. If this counter ever reaches below zero, this + function emits a fatal error (via :c:func:`Py_FatalError`). + + If the attached thread state is owned by ``PyThreadState_Ensure``, then the + attached thread state will be deallocated and deleted upon the internal counter + reaching zero. Otherwise, nothing happens when the counter reaches zero. + + .. versionadded:: 3.15 + +.. c:type:: PyThreadStateToken + + An opaque token retrieved from a :c:func:`PyThreadState_Ensure` call + and passed to a corresponding :c:func:`PyThreadState_Release` call. + + +.. _legacy-api: .. _gilstate: -Legacy API ----------- +GIL-state APIs +-------------- -Another common pattern to call Python code from a non-Python thread is to use -:c:func:`PyGILState_Ensure` followed by a call to :c:func:`PyGILState_Release`. +The following APIs are generally not compatible with subinterpreters and +will hang the process during interpreter finalization (see +:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were +:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs +`. -These functions do not work well when multiple interpreters exist in the Python -process. If no Python interpreter has ever been used in the current thread (which -is common for threads created outside Python), ``PyGILState_Ensure`` will create -and attach a thread state for the "main" interpreter (the first interpreter in -the Python process). -Additionally, these functions have thread-safety issues during interpreter -finalization. Using ``PyGILState_Ensure`` during finalization will likely -crash the process. +.. c:type:: PyGILState_STATE -Usage of these functions look like such:: + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); + The GIL was already held when :c:func:`PyGILState_Ensure` was called. - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. + + This function has no way to return an error. As such, errors are either fatal + (that is, they send ``SIGABRT`` and crash the process; see + :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as + during interpreter finalization). + + .. warning:: + Calling this function when the interpreter is finalizing will + infinitely hang the thread, which may cause deadlocks. + :ref:`cautions-regarding-runtime-finalization` for more details. + + In addition, this function generally does not work with subinterpreters + when used from foreign threads, because this function has no way of + knowing which interpreter created the thread (and as such, will implicitly + pick the main interpreter). + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` instead. + + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GIL-state API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Release` instead. + + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`thread state` that was most recently :term:`attached + ` for this thread. (If the most recent thread state + has been deleted, this returns ``NULL``.) + + If the caller has an attached thread state, it is returned. + + In other terms, this function returns the thread state that will be used by + :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then + ``PyGILState_Ensure`` will create a new thread state. + + This function cannot fail. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + instead. + + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread has an :term:`attached thread state` + that matches the thread state returned by + :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread + state or it otherwise doesn't match, then this returns ``0``. + + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. + + This is mainly a helper/diagnostic function. + + .. versionadded:: 3.4 - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); + .. soft-deprecated:: 3.15 + Use ``PyThreadState_GetUnchecked() != NULL`` instead. .. _fork-and-threads: @@ -398,101 +607,6 @@ C extensions. thread if the runtime is finalizing. -GIL-state APIs --------------- - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:type:: PyGILState_STATE - - The type of the value returned by :c:func:`PyGILState_Ensure` and passed to - :c:func:`PyGILState_Release`. - - .. c:enumerator:: PyGILState_LOCKED - - The GIL was already held when :c:func:`PyGILState_Ensure` was called. - - .. c:enumerator:: PyGILState_UNLOCKED - - The GIL was not held when :c:func:`PyGILState_Ensure` was called. - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. Failure is a fatal error. - - .. warning:: - Calling this function when the runtime is finalizing is unsafe. Doing - so will either hang the thread until the program ends, or fully crash - the interpreter in rare cases. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`attached thread state` for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - .. note:: - This function may return non-``NULL`` even when the :term:`thread state` - is detached. - Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. seealso:: :c:func:`PyThreadState_Get` - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its :term:`thread state ` initialized - via :c:func:`PyGILState_Ensure` will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the :term:`GIL` is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. note:: - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. versionadded:: 3.4 - - Low-level APIs -------------- @@ -622,10 +736,10 @@ Low-level APIs .. c:function:: PyObject* PyThreadState_GetDict() Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in + information. Each extension should use a unique key to store a state in the dictionary. It is okay to call this function when no :term:`thread state` is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no + ``NULL`` and no exception has been raised, then the caller should assume no thread state is attached. @@ -704,7 +818,7 @@ pointer and a void pointer argument. possible. If the main thread is busy executing a system call, *func* won't be called before the system call returns. This function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. + arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. .. versionadded:: 3.1 diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index c9bb5c3f09ac186..48eb16bd90834ba 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -3,7 +3,7 @@ .. _typeobjects: Type Objects ------------- +============ .. index:: pair: object; type @@ -110,11 +110,16 @@ Type Objects :c:func:`!_PyType_Lookup` is not called on *type* between the modifications; this is an implementation detail and subject to change.) + The callback is also invoked when a watched heap type is deallocated. + An extension should never call ``PyType_Watch`` with a *watcher_id* that was not returned to it by a previous call to :c:func:`PyType_AddWatcher`. .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The callback is now also invoked when a watched heap type is deallocated. + .. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) @@ -138,8 +143,17 @@ Type Objects called on *type* or any type in its MRO; violating this rule could cause infinite recursion. + The callback may be called during type deallocation. In this case, the type + object is temporarily resurrected (its reference count is at least 1) and all + its attributes are still valid. However, the callback should not store new + strong references to the type, as this would resurrect the object and prevent + its deallocation. + .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The callback may now be called during deallocation of a watched heap type. + .. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) @@ -384,36 +398,19 @@ Type Objects * :py:mod:`weakref` -Creating Heap-Allocated Types -............................. - -The following functions and structs are used to create -:ref:`heap types `. - -.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) - - Create and return a :ref:`heap type ` from the *spec* - (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). +.. _creating-heap-types: - The metaclass *metaclass* is used to construct the resulting type object. - When *metaclass* is ``NULL``, the metaclass is derived from *bases* - (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). +Creating Heap-Allocated Types +----------------------------- - Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not - supported, except if ``tp_new`` is ``NULL``. +The following function is used to create :ref:`heap types `: - The *bases* argument can be used to specify base classes; it can either - be only one class or a tuple of classes. - If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. - If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. - If that also is ``NULL``, the new type derives from :class:`object`. +.. c:function:: PyObject *PyType_FromSlots(const PySlot *slots) - The *module* argument can be used to record the module in which the new - class is defined. It must be a module object or ``NULL``. - If not ``NULL``, the module is associated with the new type and can later be - retrieved with :c:func:`PyType_GetModule`. - The associated module is not inherited by subclasses; it must be specified - for each class individually. + Create and return a :ref:`heap type ` from a :c:type:`!PySlot` + array. + See :ref:`capi-slots` for general information on slots, + and :ref:`pyslot_type_slot_ids` for slots specific to type creation. This function calls :c:func:`PyType_Ready` on the new type. @@ -430,8 +427,407 @@ The following functions and structs are used to create * :py:meth:`~object.__init_subclass__` is not called on any bases. * :py:meth:`~object.__set_name__` is not called on new descriptors. + Slots are typically defined as a global static constant arrays. + However, sometimes slot values are not statically known at compile time. + For example, slots like :c:data:`Py_tp_bases`, :c:data:`Py_tp_metaclass` + and :c:data:`Py_tp_module` require live Python objects. + In this case, it is recommended to put such slots on the stack, + and use :c:macro:`Py_slot_subslots` to refer to an array of static slots. + For example:: + + static const PySlot my_slots[] = { + PySlot_STATIC_DATA(Py_tp_name, "MyClass"), + PySlot_FUNC(Py_tp_repr, my_repr_func), + ... + PySlot_END + }; + + PyObject *make_my_class(PyObject *module) { + PySlot all_slots[] = { + PySlot_STATIC_DATA(Py_slot_subslots, my_slots), + PySlot_DATA(Py_tp_module, module), + PySlot_END + }; + return PyType_FromSlots(all_slots); + } + +Heap types created without the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag may be +modified, for example by setting attributes on them, as with classes defined +in Python code. +Sometimes, such modifications are necessary to fully initialize a type, +but you may wish to prevent users from changing the type after +the initialization is done: + +.. c:function:: int PyType_Freeze(PyTypeObject *type) + + Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. + + All base classes of *type* must be immutable. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + The type must not be used before it's made immutable. For example, type + instances must not be created before the type is made immutable. + + .. versionadded:: 3.14 + + +.. _pyslot_type_slot_ids: + +Type slot IDs +............. + +Most type slot IDs are named like the field names of the structures +:c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, +:c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and +:c:type:`PyAsyncMethods` with an added ``Py_`` prefix. +For example, use: + +* :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` +* :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` +* :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` + +The following slots need additional considerations when specified as slots: + +* :c:data:`Py_tp_name` +* :c:data:`Py_tp_basicsize` and :c:data:`Py_tp_extra_basicsize` +* :c:data:`Py_tp_itemsize` +* :c:data:`Py_tp_flags` + +Additional slots do not directly correspond to a :c:type:`!PyTypeObject` +struct field: + +* :c:data:`Py_tp_token` +* :c:data:`Py_tp_metaclass` +* :c:data:`Py_tp_module` + +The following “offset” fields cannot be set using :c:type:`PyType_Slot`: + +* :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) +* :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) +* :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in :ref:`PyMemberDef `) + +If it is not possible to switch to a ``MANAGED`` flag (for example, +for vectorcall or to support Python older than 3.12), specify the +offset in :c:data:`Py_tp_members`. +See :ref:`PyMemberDef documentation ` +for details. + +The following internal fields cannot be set at all when creating a heap +type: + +* :c:member:`~PyTypeObject.tp_dict`, + :c:member:`~PyTypeObject.tp_mro`, + :c:member:`~PyTypeObject.tp_cache`, + :c:member:`~PyTypeObject.tp_subclasses`, and + :c:member:`~PyTypeObject.tp_weaklist`. + +The :c:data:`Py_tp_base` slot is equivalent to :c:data:`Py_tp_bases`; +both may be set either to a type or a tuple of types. +If both are specified, the value of :c:data:`Py_tp_bases` +is used. + +Slot values may not be ``NULL``, except for the following: + +* :c:data:`Py_tp_doc` +* :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` + rather than ``NULL``) + +.. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + +.. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API `. + +.. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set + using :c:data:`Py_tp_vectorcall`. See the field's documentation + for details. + +.. versionchanged:: 3.15 + The :c:data:`Py_tp_bases` slot may be set to a single type object, + making it equivalent to the :c:data:`Py_tp_base` slot. + Previously, a tuple of types was required. + +The following slots correspond to fields in the underlying type structure, +but need extra remarks for use as slots: + +.. c:macro:: Py_tp_name + + :c:member:`Slot ID ` for the name of the type, + used to set :c:member:`PyTypeObject.tp_name`. + + This slot (or :c:member:`PyType_Spec.name`) is required to create a type. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.name` instead. + + .. impl-detail:: + + CPython processes slots in order. + It is recommended to put ``Py_tp_name`` at the beginning of the slots + array, so that if processing of a later slots fails, error messages + can include the name. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_basicsize + + :c:member:`Slot ID ` for the size of the instance in bytes. + It is used to set :c:member:`PyTypeObject.tp_basicsize`. + + The value must be positive. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:member:`PyTypeObject.tp_basicsize` instead if needed, but be aware + that a type's size is often considered an implementation detail. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_extra_basicsize + + :c:member:`Slot ID ` for type data size in bytes, that is, + how much space instances of the class need *in addition* + to space needed for superclasses. + + The value is used, together with the size of superclasses, to set + :c:member:`PyTypeObject.tp_basicsize`. + Python will insert padding as needed to meet + :c:member:`!tp_basicsize`'s alignment requirements. + + Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific + memory reserved this way. + + The value must be positive. + To specify that instances need no additional size (that is, size should be + inherited), omit the :c:macro:`!Py_tp_extra_basicsize` slot rather than + set it to zero. + + Specifying both :c:macro:`Py_tp_basicsize` and + :c:macro:`!Py_tp_extra_basicsize` is an error. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use negative :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_itemsize + + :c:member:`Slot ID ` for the size of one element of a + variable-size type, in bytes. + Used to set :c:member:`PyTypeObject.tp_itemsize`. + See :c:member:`!tp_itemsize` documentation for caveats. + + The value must be positive. + + If this slot is missing, :c:member:`~PyTypeObject.tp_itemsize` is inherited. + Extending arbitrary variable-sized classes is dangerous, + since some types use a fixed offset for variable-sized memory, + which can then overlap fixed-sized memory used by a subclass. + To help prevent mistakes, inheriting ``itemsize`` is only possible + in the following situations: + + - The base is not variable-sized (its + :c:member:`~PyTypeObject.tp_itemsize`). + - The requested :c:member:`PyType_Spec.basicsize` is positive, + suggesting that the memory layout of the base class is known. + - The requested :c:member:`PyType_Spec.basicsize` is zero, + suggesting that the subclass does not access the instance's memory + directly. + - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.itemsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_flags + + :c:member:`Slot ID ` for type flags, used to set + :c:member:`PyTypeObject.tp_flags`. + + The ``Py_TPFLAGS_HEAPTYPE`` flag is not set, + :c:func:`PyType_FromSpecWithBases` sets it automatically. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use negative :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`PyType_GetFlags` instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_bases + + :c:member:`Slot ID ` for type flags, used to set + :c:member:`PyTypeObject.tp_bases`. + + The slot can be set to a tuple of type objects which the newly created + type should inherit from, like the "positional arguments" of + a Python :ref:`class definition `. + + Alternately, the slot can be set to a single type object to specify + a single base. + The effect is the same as specifying a one-element tuple. + + .. versionchanged:: 3.15 + + Previously, :c:macro:`!Py_tp_bases` required a tuple of types. + +.. c:macro:: Py_tp_base + + Equivalent to :c:macro:`Py_tp_bases` (with ``s`` at the end). + If both are specified, :c:macro:`!Py_tp_bases` takes priority and + this slot is ignored. + + .. versionchanged:: 3.15 + + Previously, :c:macro:`!Py_tp_base` required a single type, not a tuple. + + .. soft-deprecated:: 3.15 + + When not targetting older Python versions, pefer :c:macro:`!Py_tp_bases`. + +The following slots do not correspond to public fields in the +underlying structures: + +.. c:macro:: Py_tp_metaclass + + :c:member:`Slot ID ` for the metaclass used to construct + the resulting type object. + When omitted the metaclass is derived from bases + (:c:macro:`Py_tp_bases` or the *bases* argument of + :c:func:`PyType_FromMetaclass`). + + Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not + supported, except if ``tp_new`` is ``NULL``. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:func:`PyType_FromMetaclass` to specify a metaclass with + :c:type:`!PyType_Spec`. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`Py_TYPE` on the type object instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_module + + :c:member:`Slot ID ` for recording the module in which + the new class is defined. + + The value must be a module object. + The module is associated with the new type and can later be + retrieved with :c:func:`PyType_GetModule`. + The associated module is not inherited by subclasses; it must be specified + for each class individually. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:func:`PyType_FromMetaclass` to specify a module with + :c:type:`!PyType_Spec`. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`PyType_GetModule` instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_token + + :c:member:`Slot ID ` for recording a static memory layout ID + for a class. + + If the class is defined using a :c:type:`PyType_Spec`, and that spec is + statically allocated, the token can be set to the spec using the special + value :c:data:`Py_TP_USE_SPEC`: + + .. code-block:: c + + static PyType_Slot foo_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + + It can also be set to an arbitrary pointer, but you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + + Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has + a given token -- that is, check whether the memory layout is compatible. + + To get the token for a given class (without considering superclasses), + use :c:func:`PyType_GetSlot` with ``Py_tp_token``. + + .. versionadded:: 3.14 + + .. c:namespace:: NULL + + .. c:macro:: Py_TP_USE_SPEC + + Used as a value with :c:data:`Py_tp_token` to set the token to the + class's :c:type:`PyType_Spec`. + May only be used for classes defined using :c:type:`!PyType_Spec`. + + Expands to ``NULL``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_tp_slots + + :c:member:`Slot ID ` that works like + :c:macro:`Py_slot_subslots`, except it specifies an array of + :c:type:`PyType_Slot` structures. + + .. versionadded:: 3.15 + + +Soft-deprecated API +------------------- + +The following functions are :term:`soft deprecated`. +They will continue to work, but new features will be added as slots for +:c:func:`PyType_FromSlots`, not as arguments to new ``PyType_From*`` functions. + +.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) + + Create and return a :ref:`heap type ` from the *spec* + (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). + + A non-``NULL`` *metaclass* argument corresponds to the + :c:macro:`Py_tp_metaclass` slot. + + A non-``NULL`` *bases* argument corresponds to the :c:data:`Py_tp_bases` + slot, and takes precedence over :c:data:`Py_tp_bases` and + :c:data:`Py_tp_bases` slots. + + A non-``NULL`` *module* argument corresponds to the + :c:macro:`Py_tp_module` slot. + + This function calls :c:func:`PyType_Ready` on the new type. + + Note that this function does *not* fully match the behavior of + calling :py:class:`type() ` or using the :keyword:`class` statement. + See the note in :c:func:`PyType_FromSlots` documentation for details. + .. versionadded:: 3.12 + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) @@ -459,6 +855,10 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -481,6 +881,10 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) @@ -502,20 +906,9 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next -.. c:function:: int PyType_Freeze(PyTypeObject *type) - - Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. - - All base classes of *type* must be immutable. - - On success, return ``0``. - On error, set an exception and return ``-1``. - - The type must not be used before it's made immutable. For example, type - instances must not be created before the type is made immutable. - - .. versionadded:: 3.14 + Prefer :c:func:`PyType_FromSlots` in new code. .. raw:: html @@ -528,27 +921,23 @@ The following functions and structs are used to create .. c:type:: PyType_Spec - Structure defining a type's behavior. + Structure defining a type's behavior, used for soft-deprecated functions + like :c:func:`PyType_FromMetaclass`. + + This structure contains several members that can instead be specified + as :ref:`slots ` for :c:func:`PyType_FromSlots`, + and an array of slot entries with a simpler structure. .. c:member:: const char* name - Name of the type, used to set :c:member:`PyTypeObject.tp_name`. + Corresponds to :c:macro:`Py_tp_name`. .. c:member:: int basicsize - If positive, specifies the size of the instance in bytes. - It is used to set :c:member:`PyTypeObject.tp_basicsize`. - - If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize` - should be inherited. + If positive, corresponds to :c:macro:`Py_tp_basicsize`. - If negative, the absolute value specifies how much space instances of the - class need *in addition* to the superclass. - Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific - memory reserved this way. - For negative :c:member:`!basicsize`, Python will insert padding when - needed to meet :c:member:`~PyTypeObject.tp_basicsize`'s alignment - requirements. + If negative, corresponds to :c:macro:`Py_tp_extra_basicsize` set to + the absolute value. .. versionchanged:: 3.12 @@ -556,160 +945,53 @@ The following functions and structs are used to create .. c:member:: int itemsize - Size of one element of a variable-size type, in bytes. - Used to set :c:member:`PyTypeObject.tp_itemsize`. - See ``tp_itemsize`` documentation for caveats. - - If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited. - Extending arbitrary variable-sized classes is dangerous, - since some types use a fixed offset for variable-sized memory, - which can then overlap fixed-sized memory used by a subclass. - To help prevent mistakes, inheriting ``itemsize`` is only possible - in the following situations: - - - The base is not variable-sized (its - :c:member:`~PyTypeObject.tp_itemsize`). - - The requested :c:member:`PyType_Spec.basicsize` is positive, - suggesting that the memory layout of the base class is known. - - The requested :c:member:`PyType_Spec.basicsize` is zero, - suggesting that the subclass does not access the instance's memory - directly. - - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. + Corresponds to :c:macro:`Py_tp_itemsize`. .. c:member:: unsigned int flags - Type flags, used to set :c:member:`PyTypeObject.tp_flags`. - - If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, - :c:func:`PyType_FromSpecWithBases` sets it automatically. + Corresponds to :c:macro:`Py_tp_flags`. .. c:member:: PyType_Slot *slots - Array of :c:type:`PyType_Slot` structures. - Terminated by the special slot value ``{0, NULL}``. + Array of :c:type:`PyType_Slot` (not :c:type:`PySlot`) structures. + Terminated by the special slot value ``{0, NULL}``. Each slot ID should be specified at most once. -.. raw:: html - - - - - -.. c:type:: PyType_Slot - - Structure defining optional functionality of a type, containing a slot ID - and a value pointer. - - .. c:member:: int slot - - A slot ID. - - Slot IDs are named like the field names of the structures - :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, - :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and - :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. - For example, use: - - * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` - * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` - * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` + .. c:namespace:: NULL - An additional slot is supported that does not correspond to a - :c:type:`!PyTypeObject` struct field: + .. raw:: html - * :c:data:`Py_tp_token` + + + - The following “offset” fields cannot be set using :c:type:`PyType_Slot`: + .. c:type:: PyType_Slot - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (use ``"__vectorcalloffset__"`` in - :ref:`PyMemberDef `) + Structure defining optional functionality of a type, used for + soft-deprecated functions like :c:func:`PyType_FromMetaclass`. - If it is not possible to switch to a ``MANAGED`` flag (for example, - for vectorcall or to support Python older than 3.12), specify the - offset in :c:data:`Py_tp_members`. - See :ref:`PyMemberDef documentation ` - for details. + Note that a :c:type:`!PyType_Slot` array may be included in a + :c:type:`!PySlot` array using :c:macro:`Py_tp_slots`, + and vice versa using :c:macro:`Py_slot_subslots`. - The following internal fields cannot be set at all when creating a heap - type: + Each :c:type:`!PyType_Slot` structure ``tpslot`` is interpreted + as the following :c:type:`PySlot` structure:: - * :c:member:`~PyTypeObject.tp_dict`, - :c:member:`~PyTypeObject.tp_mro`, - :c:member:`~PyTypeObject.tp_cache`, - :c:member:`~PyTypeObject.tp_subclasses`, and - :c:member:`~PyTypeObject.tp_weaklist`. + (PySlot){ + .sl_id=tpslot.slot, + .sl_flags=PySlot_INTPTR | sub_static, + .sl_ptr=tpslot.func + } - Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be - problematic on some platforms. - To avoid issues, use the *bases* argument of - :c:func:`PyType_FromSpecWithBases` instead. + where ``sub_static`` is ``PySlot_STATIC`` if the slot requires + the flag (such as for :c:macro:`Py_tp_methods`), or if this flag + is present on the "parent" :c:macro:`!Py_tp_slots` slot (if any). - .. versionchanged:: 3.9 - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + .. c:member:: int slot - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. + Corresponds to :c:member:`PySlot.sl_id`. - .. versionchanged:: 3.14 - The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set - using :c:data:`Py_tp_vectorcall`. See the field's documentation - for details. + .. c:member:: void *pfunc - .. c:member:: void *pfunc - - The desired value of the slot. In most cases, this is a pointer - to a function. - - *pfunc* values may not be ``NULL``, except for the following slots: - - * :c:data:`Py_tp_doc` - * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` - rather than ``NULL``) - - -.. c:macro:: Py_tp_token - - A :c:member:`~PyType_Slot.slot` that records a static memory layout ID - for a class. - - If the :c:type:`PyType_Spec` of the class is statically - allocated, the token can be set to the spec using the special value - :c:data:`Py_TP_USE_SPEC`: - - .. code-block:: c - - static PyType_Slot foo_slots[] = { - {Py_tp_token, Py_TP_USE_SPEC}, - - It can also be set to an arbitrary pointer, but you must ensure that: - - * The pointer outlives the class, so it's not reused for something else - while the class exists. - * It "belongs" to the extension module where the class lives, so it will not - clash with other extensions. - - Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has - a given token -- that is, check whether the memory layout is compatible. - - To get the token for a given class (without considering superclasses), - use :c:func:`PyType_GetSlot` with ``Py_tp_token``. - - .. versionadded:: 3.14 - - .. c:namespace:: NULL - - .. c:macro:: Py_TP_USE_SPEC - - Used as a value with :c:data:`Py_tp_token` to set the token to the - class's :c:type:`PyType_Spec`. - Expands to ``NULL``. - - .. versionadded:: 3.14 + Corresponds to :c:member:`PySlot.sl_ptr`. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 98fe68737deb81c..ec2fba6da8b0438 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -31,7 +31,7 @@ two types exist -- :ref:`GenericAlias ` and static PyMethodDef my_obj_methods[] = { // Other methods. ... - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "my_obj is generic over its contained type"} ... } diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index d3d8239365f9bf9..16dcb880712d244 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -555,6 +555,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_name + See :c:macro:`Py_tp_name` for the corresponding + :c:member:`Slot ID `. + Pointer to a NUL-terminated string containing the name of the type. For types that are accessible as module globals, the string should be the full module name, followed by a dot, followed by the type name; for built-in types, it @@ -594,6 +597,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) These fields allow calculating the size in bytes of instances of the type. + See :c:macro:`Py_tp_basicsize`, :c:macro:`Py_tp_extra_basicsize` and + :c:macro:`Py_tp_itemsize` for the corresponding + :c:member:`Slot IDs `. + There are two kinds of types: types with fixed-length instances have a zero :c:member:`!tp_itemsize` field, types with variable-length instances have a non-zero :c:member:`!tp_itemsize` field. For a type with fixed-length instances, all @@ -1133,6 +1140,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: unsigned long PyTypeObject.tp_flags + See :c:macro:`Py_tp_flags` for the corresponding + :c:member:`Slot ID `. + This field is a bit mask of various flags. Some flags indicate variant semantics for certain situations; others are used to indicate that certain fields in the type object (or in the extension structures referenced via @@ -1926,12 +1936,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base - .. corresponding-type-slot:: Py_tp_base - An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. + For the corresponding slot ID, see :c:macro:`Py_tp_base`. + .. note:: .. from Modules/xxmodule.c @@ -2243,17 +2253,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases - .. corresponding-type-slot:: Py_tp_bases - Tuple of base types. This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the :c:data:`Py_tp_bases` - :c:type:`slot ` can be used instead of the *bases* argument - of :c:func:`PyType_FromSpecWithBases`. - The argument form is preferred. + For the corresponding slot ID, see :c:macro:`Py_tp_bases`. .. warning:: @@ -2965,13 +2970,13 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :exc:`BufferError`, - set :c:expr:`view->obj` to ``NULL`` and return ``-1``. + set ``view->obj`` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. (3) Increment an internal counter for the number of exports. - (4) Set :c:expr:`view->obj` to *exporter* and increment :c:expr:`view->obj`. + (4) Set ``view->obj`` to *exporter* and increment ``view->obj``. (5) Return ``0``. @@ -2997,10 +3002,10 @@ Buffer Object Structures schemes can be used: * Re-export: Each member of the tree acts as the exporting object and - sets :c:expr:`view->obj` to a new reference to itself. + sets ``view->obj`` to a new reference to itself. * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:expr:`view->obj` will be a new reference to the root + tree. Here, ``view->obj`` will be a new reference to the root object. The individual fields of *view* are described in section @@ -3054,7 +3059,7 @@ Buffer Object Structures *view* argument. - This function MUST NOT decrement :c:expr:`view->obj`, since that is + This function MUST NOT decrement ``view->obj``, since that is done automatically in :c:func:`PyBuffer_Release` (this scheme is useful for breaking reference cycles). diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 059a7ef399ae0f5..634dcbce7a57915 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -762,7 +762,7 @@ APIs: The string must not have been “used” yet. See :c:func:`PyUnicode_New` for details. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -1174,7 +1174,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1196,7 +1196,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs @@ -1953,7 +1953,7 @@ The following API is deprecated. whether you selected a "narrow" or "wide" Unicode version of Python at build time. - .. deprecated-removed:: 3.13 3.15 + .. deprecated-removed:: 3.13 3.16 .. c:function:: int PyUnicode_READY(PyObject *unicode) diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index db6ae0a9d4ea3d7..8762a003c5218d3 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -43,7 +43,11 @@ as much as it can. should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly referenceable object, or if *callback* is not callable, ``None``, or - ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + ``NULL``, this will raise :exc:`TypeError` and return ``NULL``. + + .. versionchanged:: next + Raise :exc:`!TypeError` if *callback* is not callable, ``None``, or + ``NULL``. .. seealso:: :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly @@ -59,7 +63,11 @@ as much as it can. collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly referenceable object, or if *callback* is not callable, - ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + ``NULL``, this will raise :exc:`TypeError` and return ``NULL``. + + .. versionchanged:: next + Raise :exc:`!TypeError` if *callback* is not callable, ``None``, or + ``NULL``. .. seealso:: :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly diff --git a/Doc/conf.py b/Doc/conf.py index e2dff74538a3422..9b103a594b235cf 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -8,15 +8,13 @@ import os import sys -from importlib import import_module from importlib.util import find_spec # Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) -# Python specific content from Doc/Tools/extensions/pyspecific.py -from pyspecific import SOURCE_URI +from patchlevel import get_header_version_info, get_version_info # General configuration # --------------------- @@ -78,7 +76,7 @@ # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py -version, release = import_module('patchlevel').get_version_info() +version, release = get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` @@ -349,8 +347,6 @@ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} } -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', # The paper size ('letterpaper' or 'a4paper'). @@ -557,16 +553,20 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] + # Options for sphinx.ext.extlinks # ------------------------------- +v = get_header_version_info() +branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" + # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), - "source": (SOURCE_URI, "%s"), + "source": (f"https://github.com/python/cpython/tree/{branch}/%s", "%s"), } extlinks_detect_hardcoded_links = True diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 663b79e45eec17d..60c02aabeb89c51 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2040,6 +2040,7 @@ PySeqIter_New:PyObject*:seq:0: PySentinel_New:PyObject*::+1: PySentinel_New:const char*:name:: PySentinel_New:const char*:module_name:: +PySentinel_New:const char*:repr:: PySequence_Check:int::: PySequence_Check:PyObject*:o:0: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 2ef2bccf2b7728c..86080fac7163838 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,7 +1,7 @@ role,name,added,ifdef_note,struct_abi_kind macro,METH_CLASS,3.2,, macro,METH_COEXIST,3.2,, -macro,METH_FASTCALL,3.7,, +macro,METH_FASTCALL,3.10,, macro,METH_METHOD,3.7,, macro,METH_NOARGS,3.2,, macro,METH_O,3.2,, @@ -129,6 +129,12 @@ func,PyComplex_FromDoubles,3.2,, func,PyComplex_ImagAsDouble,3.2,, func,PyComplex_RealAsDouble,3.2,, data,PyComplex_Type,3.2,, +type,PyCriticalSection,3.15,,full-abi +type,PyCriticalSection2,3.15,,full-abi +func,PyCriticalSection2_Begin,3.15,, +func,PyCriticalSection2_End,3.15,, +func,PyCriticalSection_Begin,3.15,, +func,PyCriticalSection_End,3.15,, func,PyDescr_NewClassMethod,3.2,, func,PyDescr_NewGetSet,3.2,, func,PyDescr_NewMember,3.2,, @@ -363,6 +369,10 @@ func,PyImport_ImportModuleLevel,3.2,, func,PyImport_ImportModuleLevelObject,3.7,, func,PyImport_ReloadModule,3.2,, func,PyIndex_Check,3.8,, +type,PyInterpreterGuard,3.15,,opaque +func,PyInterpreterGuard_Close,3.15,, +func,PyInterpreterGuard_FromCurrent,3.15,, +func,PyInterpreterGuard_FromView,3.15,, type,PyInterpreterState,3.2,,opaque func,PyInterpreterState_Clear,3.2,, func,PyInterpreterState_Delete,3.2,, @@ -370,6 +380,10 @@ func,PyInterpreterState_Get,3.9,, func,PyInterpreterState_GetDict,3.8,, func,PyInterpreterState_GetID,3.7,, func,PyInterpreterState_New,3.2,, +type,PyInterpreterView,3.15,,opaque +func,PyInterpreterView_Close,3.15,, +func,PyInterpreterView_FromCurrent,3.15,, +func,PyInterpreterView_FromMain,3.15,, func,PyIter_Check,3.8,, func,PyIter_Next,3.2,, func,PyIter_NextItem,3.14,, @@ -670,6 +684,19 @@ func,PySlice_GetIndicesEx,3.2,, func,PySlice_New,3.2,, data,PySlice_Type,3.2,, func,PySlice_Unpack,3.7,, +type,PySlot,3.15,,full-abi +macro,PySlot_DATA,3.15,, +macro,PySlot_END,3.15,, +macro,PySlot_FUNC,3.15,, +macro,PySlot_INT64,3.15,, +macro,PySlot_INTPTR,3.15,, +macro,PySlot_OPTIONAL,3.15,, +macro,PySlot_PTR,3.15,, +macro,PySlot_PTR_STATIC,3.15,, +macro,PySlot_SIZE,3.15,, +macro,PySlot_STATIC,3.15,, +macro,PySlot_STATIC_DATA,3.15,, +macro,PySlot_UINT64,3.15,, func,PyState_AddModule,3.3,, func,PyState_FindModule,3.2,, func,PyState_RemoveModule,3.3,, @@ -697,14 +724,18 @@ func,PySys_SetObject,3.2,, func,PySys_WriteStderr,3.2,, func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque +type,PyThreadStateToken,3.15,,opaque func,PyThreadState_Clear,3.2,, func,PyThreadState_Delete,3.2,, +func,PyThreadState_Ensure,3.15,, +func,PyThreadState_EnsureFromView,3.15,, func,PyThreadState_Get,3.2,, func,PyThreadState_GetDict,3.2,, func,PyThreadState_GetFrame,3.10,, func,PyThreadState_GetID,3.10,, func,PyThreadState_GetInterpreter,3.10,, func,PyThreadState_New,3.2,, +func,PyThreadState_Release,3.15,, func,PyThreadState_SetAsyncExc,3.2,, func,PyThreadState_Swap,3.2,, func,PyThread_GetInfo,3.3,, @@ -749,6 +780,7 @@ func,PyType_ClearCache,3.2,, func,PyType_Freeze,3.14,, func,PyType_FromMetaclass,3.12,, func,PyType_FromModuleAndSpec,3.10,, +func,PyType_FromSlots,3.15,, func,PyType_FromSpec,3.2,, func,PyType_FromSpecWithBases,3.3,, func,PyType_GenericAlloc,3.2,, @@ -906,6 +938,8 @@ macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, +macro,Py_BEGIN_CRITICAL_SECTION,3.15,, +macro,Py_BEGIN_CRITICAL_SECTION2,3.15,, macro,Py_BLOCK_THREADS,3.2,, func,Py_BuildValue,3.2,, func,Py_BytesMain,3.8,, @@ -913,6 +947,8 @@ func,Py_CompileString,3.2,, func,Py_DecRef,3.2,, func,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, +macro,Py_END_CRITICAL_SECTION,3.15,, +macro,Py_END_CRITICAL_SECTION2,3.15,, func,Py_EncodeLocale,3.7,, func,Py_EndInterpreter,3.2,, func,Py_EnterRecursiveCall,3.9,, @@ -1008,6 +1044,7 @@ macro,Py_mod_gil,3.13,, macro,Py_mod_methods,3.15,, macro,Py_mod_multiple_interpreters,3.12,, macro,Py_mod_name,3.15,, +macro,Py_mod_slots,3.15,, macro,Py_mod_state_clear,3.15,, macro,Py_mod_state_free,3.15,, macro,Py_mod_state_size,3.15,, @@ -1051,6 +1088,9 @@ macro,Py_nb_rshift,3.2,, macro,Py_nb_subtract,3.2,, macro,Py_nb_true_divide,3.2,, macro,Py_nb_xor,3.2,, +macro,Py_slot_end,3.15,, +macro,Py_slot_invalid,3.15,, +macro,Py_slot_subslots,3.15,, macro,Py_sq_ass_item,3.2,, macro,Py_sq_concat,3.2,, macro,Py_sq_contains,3.2,, @@ -1063,6 +1103,7 @@ type,Py_ssize_t,3.2,, macro,Py_tp_alloc,3.2,, macro,Py_tp_base,3.2,, macro,Py_tp_bases,3.2,, +macro,Py_tp_basicsize,3.15,, macro,Py_tp_call,3.2,, macro,Py_tp_clear,3.2,, macro,Py_tp_dealloc,3.2,, @@ -1070,7 +1111,9 @@ macro,Py_tp_del,3.2,, macro,Py_tp_descr_get,3.2,, macro,Py_tp_descr_set,3.2,, macro,Py_tp_doc,3.2,, +macro,Py_tp_extra_basicsize,3.15,, macro,Py_tp_finalize,3.5,, +macro,Py_tp_flags,3.15,, macro,Py_tp_free,3.2,, macro,Py_tp_getattr,3.2,, macro,Py_tp_getattro,3.2,, @@ -1078,15 +1121,20 @@ macro,Py_tp_getset,3.2,, macro,Py_tp_hash,3.2,, macro,Py_tp_init,3.2,, macro,Py_tp_is_gc,3.2,, +macro,Py_tp_itemsize,3.15,, macro,Py_tp_iter,3.2,, macro,Py_tp_iternext,3.2,, macro,Py_tp_members,3.2,, +macro,Py_tp_metaclass,3.15,, macro,Py_tp_methods,3.2,, +macro,Py_tp_module,3.15,, +macro,Py_tp_name,3.15,, macro,Py_tp_new,3.2,, macro,Py_tp_repr,3.2,, macro,Py_tp_richcompare,3.2,, macro,Py_tp_setattr,3.2,, macro,Py_tp_setattro,3.2,, +macro,Py_tp_slots,3.15,, macro,Py_tp_str,3.2,, macro,Py_tp_token,3.14,, macro,Py_tp_traverse,3.2,, diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index 789ec83d2d957aa..6d0d47403ff2ed4 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -46,88 +46,3 @@ Pending removal in Python 3.15 The `pythoncapi-compat project `__ can be used to get :c:func:`PyConfig_Get` on Python 3.13 and older. - -* Functions to configure Python's initialization, deprecated in Python 3.11: - - * :c:func:`!PySys_SetArgvEx()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!PySys_SetArgv()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!Py_SetProgramName()`: - Set :c:member:`PyConfig.program_name` instead. - * :c:func:`!Py_SetPythonHome()`: - Set :c:member:`PyConfig.home` instead. - * :c:func:`!PySys_ResetWarnOptions`: - Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -* Global configuration variables: - - * :c:var:`Py_DebugFlag`: - Use :c:member:`PyConfig.parser_debug` or - :c:func:`PyConfig_Get("parser_debug") ` instead. - * :c:var:`Py_VerboseFlag`: - Use :c:member:`PyConfig.verbose` or - :c:func:`PyConfig_Get("verbose") ` instead. - * :c:var:`Py_QuietFlag`: - Use :c:member:`PyConfig.quiet` or - :c:func:`PyConfig_Get("quiet") ` instead. - * :c:var:`Py_InteractiveFlag`: - Use :c:member:`PyConfig.interactive` or - :c:func:`PyConfig_Get("interactive") ` instead. - * :c:var:`Py_InspectFlag`: - Use :c:member:`PyConfig.inspect` or - :c:func:`PyConfig_Get("inspect") ` instead. - * :c:var:`Py_OptimizeFlag`: - Use :c:member:`PyConfig.optimization_level` or - :c:func:`PyConfig_Get("optimization_level") ` instead. - * :c:var:`Py_NoSiteFlag`: - Use :c:member:`PyConfig.site_import` or - :c:func:`PyConfig_Get("site_import") ` instead. - * :c:var:`Py_BytesWarningFlag`: - Use :c:member:`PyConfig.bytes_warning` or - :c:func:`PyConfig_Get("bytes_warning") ` instead. - * :c:var:`Py_FrozenFlag`: - Use :c:member:`PyConfig.pathconfig_warnings` or - :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. - * :c:var:`Py_IgnoreEnvironmentFlag`: - Use :c:member:`PyConfig.use_environment` or - :c:func:`PyConfig_Get("use_environment") ` instead. - * :c:var:`Py_DontWriteBytecodeFlag`: - Use :c:member:`PyConfig.write_bytecode` or - :c:func:`PyConfig_Get("write_bytecode") ` instead. - * :c:var:`Py_NoUserSiteDirectory`: - Use :c:member:`PyConfig.user_site_directory` or - :c:func:`PyConfig_Get("user_site_directory") ` instead. - * :c:var:`Py_UnbufferedStdioFlag`: - Use :c:member:`PyConfig.buffered_stdio` or - :c:func:`PyConfig_Get("buffered_stdio") ` instead. - * :c:var:`Py_HashRandomizationFlag`: - Use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` or - :c:func:`PyConfig_Get("hash_seed") ` instead. - * :c:var:`Py_IsolatedFlag`: - Use :c:member:`PyConfig.isolated` or - :c:func:`PyConfig_Get("isolated") ` instead. - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: - Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or - :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. - * :c:var:`Py_LegacyWindowsStdioFlag`: - Use :c:member:`PyConfig.legacy_windows_stdio` or - :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. - * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: - Use :c:member:`PyConfig.filesystem_encoding` or - :c:func:`PyConfig_Get("filesystem_encoding") ` instead. - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: - Use :c:member:`PyConfig.filesystem_errors` or - :c:func:`PyConfig_Get("filesystem_errors") ` instead. - * :c:var:`!Py_UTF8Mode`: - Use :c:member:`PyPreConfig.utf8_mode` or - :c:func:`PyConfig_Get("utf8_mode") ` instead. - (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be - used to get these options at runtime. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst index 9453f83799c43d7..fe2d91cf316b18f 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.16.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -1,4 +1,87 @@ Pending removal in Python 3.16 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The bundled copy of ``libmpdec``. +* Functions to configure Python's initialization, deprecated in Python 3.11: + + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` or + :c:func:`PyConfig_Get("parser_debug") ` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` or + :c:func:`PyConfig_Get("verbose") ` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` or + :c:func:`PyConfig_Get("quiet") ` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` or + :c:func:`PyConfig_Get("interactive") ` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` or + :c:func:`PyConfig_Get("inspect") ` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` or + :c:func:`PyConfig_Get("optimization_level") ` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` or + :c:func:`PyConfig_Get("site_import") ` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` or + :c:func:`PyConfig_Get("bytes_warning") ` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` or + :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` or + :c:func:`PyConfig_Get("use_environment") ` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` or + :c:func:`PyConfig_Get("write_bytecode") ` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` or + :c:func:`PyConfig_Get("user_site_directory") ` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` or + :c:func:`PyConfig_Get("buffered_stdio") ` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` or + :c:func:`PyConfig_Get("hash_seed") ` instead. + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` or + :c:func:`PyConfig_Get("isolated") ` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or + :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` or + :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` or + :c:func:`PyConfig_Get("filesystem_encoding") ` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` or + :c:func:`PyConfig_Get("filesystem_errors") ` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` or + :c:func:`PyConfig_Get("utf8_mode") ` instead. + (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be + used to get these options at runtime. + +* :c:type:`Py_UNICODE` which was deprecated by :pep:`393`. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 022aee93aa70c47..820334ee43c82e4 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -40,6 +40,10 @@ Pending removal in Python 3.18 * :c:func:`!_PyUnicodeWriter_PrepareKind`: (no replacement). * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + * :c:func:`PyGen_New`: (no replacement). + * :c:func:`PyGen_NewWithQualName`: (no replacement). + * :c:func:`PyCoro_New`: (no replacement). + * :c:func:`PyAsyncGen_New`: (no replacement). The `pythoncapi-compat project `__ can be used to get diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index eedcd2e9c9dd423..6176169a6f1eba6 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -1,10 +1,6 @@ Deprecations ============ -.. include:: pending-removal-in-3.15.rst - -.. include:: pending-removal-in-3.16.rst - .. include:: pending-removal-in-3.17.rst .. include:: pending-removal-in-3.18.rst @@ -13,6 +9,8 @@ Deprecations .. include:: pending-removal-in-3.20.rst +.. include:: pending-removal-in-3.21.rst + .. include:: pending-removal-in-future.rst .. include:: soft-deprecations.rst @@ -20,10 +18,6 @@ Deprecations C API deprecations ------------------ -.. include:: c-api-pending-removal-in-3.15.rst - -.. include:: c-api-pending-removal-in-3.16.rst - .. include:: c-api-pending-removal-in-3.18.rst .. include:: c-api-pending-removal-in-3.19.rst diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index 1d9a3095813a6de..5c0e592f4caeda1 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -8,11 +8,6 @@ Pending removal in Python 3.15 is deprecated. In Python 3.15, ``__cached__`` will cease to be set or take into consideration by the import system or standard library. (:gh:`97879`) - * Setting :attr:`~module.__package__` on a module while - failing to set :attr:`__spec__.parent ` - is deprecated. In Python 3.15, :attr:`!__package__` will cease to be set or - take into consideration by the import system or standard library. (:gh:`97879`) - * :mod:`ctypes`: * The undocumented :func:`!ctypes.SetPointerType` function @@ -82,7 +77,7 @@ Pending removal in Python 3.15 Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` to create a TypedDict with zero field. - * The :func:`!typing.no_type_check_decorator` decorator function + * The :deco:`!typing.no_type_check_decorator` decorator function has been deprecated since Python 3.13. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index b00c7002b03772d..5a28cc766a95961 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -8,6 +8,13 @@ Pending removal in Python 3.16 is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or taken into consideration by the import system or the standard library. + * Setting :attr:`~module.__package__` on a module while + failing to set :attr:`__spec__.parent ` + is deprecated. In Python 3.16, :attr:`!__package__` will cease to be + taken into consideration by the import system or standard library. (:gh:`97879`) + +* The bundled copy of ``libmpdec``. + * :mod:`array`: * The ``'u'`` format code (:c:type:`wchar_t`) @@ -84,12 +91,12 @@ Pending removal in Python 3.16 * :mod:`symtable`: - * The :meth:`Class.get_methods ` method + * The :meth:`!symtable.Class.get_methods` method has been deprecated since Python 3.14. * :mod:`sys`: - * The :func:`~sys._enablelegacywindowsfsencoding` function + * The :func:`!_enablelegacywindowsfsencoding` function has been deprecated since Python 3.13. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. @@ -101,5 +108,5 @@ Pending removal in Python 3.16 * :mod:`tarfile`: - * The undocumented and unused :attr:`!TarFile.tarfile` attribute + * The undocumented and unused :attr:`!TarInfo.tarfile` attribute has been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 952ffad64356d9b..d6e4e81afbbd639 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -37,6 +37,11 @@ Pending removal in Python 3.17 is deprecated and scheduled for removal in Python 3.17. (Contributed by Stan Ulbrych in :gh:`136702`.) +* :mod:`webbrowser`: + + - :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS`. (:gh:`137586`) + * :mod:`typing`: - Before Python 3.14, old-style unions were implemented using the private class @@ -63,3 +68,13 @@ Pending removal in Python 3.17 See :pep:`PEP 688 <688#current-options>` for more details. (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`tkinter`: + + - The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, + :meth:`!trace` (an alias of :meth:`!trace_variable`), + :meth:`!trace_vdelete` and :meth:`!trace_vinfo`, deprecated since + Python 3.14, are scheduled for removal in Python 3.17. + Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` + instead. + (Contributed by Serhiy Storchaka in :gh:`120220`.) diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 044bb8a3934a2a7..4a58c606ab7596e 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -31,3 +31,12 @@ Pending removal in Python 3.19 * :meth:`http.cookies.BaseCookie.js_output` is deprecated and will be removed in Python 3.19. +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does not automatically close the current file. + + Before Python 3.14, this property was used to implement the corresponding + ``read()`` and ``readline()`` methods for :class:`~imaplib.IMAP4` but this + is no longer the case since then. diff --git a/Doc/deprecations/pending-removal-in-3.21.rst b/Doc/deprecations/pending-removal-in-3.21.rst new file mode 100644 index 000000000000000..18b89a20e4a2082 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.21.rst @@ -0,0 +1,19 @@ +Pending removal in Python 3.21 +------------------------------ + +* :mod:`abc` + + * Soft-deprecated since Python 3.3 :class:`abc.abstractclassmethod`, + :class:`abc.abstractstaticmethod`, and :class:`abc.abstractproperty` + now raise a :exc:`DeprecationWarning`. + These classes will be removed in Python 3.21, instead + use :func:`abc.abstractmethod` with :func:`classmethod`, + :func:`staticmethod`, and :class:`property` respectively. + +* :mod:`ast`: + + * Classes ``slice``, ``Index``, ``ExtSlice``, ``Suite``, ``Param``, + ``AugLoad`` and ``AugStore``, will be removed in Python 3.21. These types + are not generated by the parser or accepted by the code generator. + * The ``dims`` property of ``ast.Tuple`` will be removed in Python 3.21. Use + the ``ast.Tuple.elts`` property instead. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index d33cbd2813d637b..110dfea8cb98abe 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -231,10 +231,8 @@ calling the Python callback functions from a C callback. Other uses are also imaginable. Fortunately, the Python interpreter is easily called recursively, and there is a -standard interface to call a Python function. (I won't dwell on how to call the -Python parser with a particular string as input --- if you're interested, have a -look at the implementation of the :option:`-c` command line option in -:file:`Modules/main.c` from the Python source code.) +standard interface to call a Python function. (If you're interested in how to call the +Python parser with a particular string as input, see :ref:`veryhigh`.) Calling a Python function is easy. First, the Python program must somehow pass you the Python function object. You should provide a function (or some other @@ -641,7 +639,7 @@ and the object is freed. An alternative strategy is called :dfn:`automatic garbage collection`. (Sometimes, reference counting is also referred to as a garbage collection -strategy, hence my use of "automatic" to distinguish the two.) The big +strategy, hence the use of "automatic" to distinguish the two.) The big advantage of automatic garbage collection is that the user doesn't need to call :c:func:`free` explicitly. (Another claimed advantage is an improvement in speed or memory usage --- this is no hard fact however.) The disadvantage is that for diff --git a/Doc/extending/first-extension-module.rst b/Doc/extending/first-extension-module.rst index cd755a98f7f5f4c..55a772e2aca24f5 100644 --- a/Doc/extending/first-extension-module.rst +++ b/Doc/extending/first-extension-module.rst @@ -164,7 +164,7 @@ Then, create ``meson.build`` containing the following: .. note:: - See `meson-python documentation `_ for details on + See the `meson-python documentation `_ for details on configuration. Now, build install the *project in the current directory* (``.``) via ``pip``: @@ -259,21 +259,25 @@ Rather than ``NULL``, the export hook should return the information needed to create a module. Let's start with the basics: the name and docstring. -The information should be defined in a ``static`` array of -:c:type:`PyModuleDef_Slot` entries, which are essentially key-value pairs. +The information should be defined in an array of +:c:type:`PySlot` entries, which are essentially key-value pairs. Define this array just before your export hook: .. code-block:: c PyABIInfo_VAR(abi_info); - static PyModuleDef_Slot spam_slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "spam"}, - {Py_mod_doc, "A wonderful module with an example function"}, - {0, NULL} + static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_STATIC_DATA(Py_mod_doc, "A wonderful module with an example function"), + PySlot_END }; +The :c:macro:`PySlot_STATIC_DATA` macro is used when the slot value +(here: ``&abi_info``, ``"spam"``, and the docstring) is a pointer to constant, +statically allocated data. + The ``PyABIInfo_VAR(abi_info);`` macro and the :c:data:`Py_mod_abi` slot are a bit of boilerplate that helps prevent extensions compiled for a different version of Python from crashing the interpreter. @@ -281,7 +285,8 @@ a different version of Python from crashing the interpreter. For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C strings -- that is, NUL-terminated, UTF-8 encoded byte arrays. -Note the zero-filled sentinel entry at the end. +Note ``PySlot_END`` sentinel entry at the end. +This marks the end of the array. If you forget it, you'll trigger undefined behavior. The array is defined as ``static`` -- that is, not visible outside this ``.c`` file. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 591565cbc013575..c2f8f72ee1f2c4b 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -2003,7 +2003,7 @@ How do I cache method calls? ---------------------------- The two principal tools for caching methods are -:func:`functools.cached_property` and :func:`functools.lru_cache`. The +:deco:`functools.cached_property` and :deco:`functools.lru_cache`. The former stores results at the instance level and the latter at the class level. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 56bc799d945e7b0..b25532d2d634126 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -96,21 +96,24 @@ Glossary :meth:`~object.__aexit__` methods. Introduced by :pep:`492`. asynchronous generator - A function which returns an :term:`asynchronous generator iterator`. It - looks like a coroutine function defined with :keyword:`async def` except - that it contains :keyword:`yield` expressions for producing a series of - values usable in an :keyword:`async for` loop. - - Usually refers to an asynchronous generator function, but may refer to an - *asynchronous generator iterator* in some contexts. In cases where the - intended meaning isn't clear, using the full terms avoids ambiguity. + Informally used to mean either an :term:`asynchronous generator + function` or an :term:`asynchronous generator iterator`, depending on + context. The formal terms :term:`asynchronous generator function` and + :term:`asynchronous generator iterator` are uncommon in practice; + "asynchronous generator" alone is almost always sufficient. + + asynchronous generator function + A function which returns an :term:`asynchronous generator iterator`. + It looks like a coroutine function defined with :keyword:`async def` + except that it contains :keyword:`yield` expressions for producing a + series of values usable in an :keyword:`async for` loop. See :pep:`525`. An asynchronous generator function may contain :keyword:`await` expressions as well as :keyword:`async for`, and :keyword:`async with` statements. asynchronous generator iterator - An object created by an :term:`asynchronous generator` function. + An object created by an :term:`asynchronous generator function`. This is an :term:`asynchronous iterator` which when called using the :meth:`~object.__anext__` method returns an awaitable object which will execute @@ -413,7 +416,7 @@ Glossary decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for - decorators are :func:`classmethod` and :func:`staticmethod`. + decorators are :deco:`classmethod` and :deco:`staticmethod`. The decorator syntax is merely syntactic sugar, the following two function definitions are semantically equivalent:: @@ -641,23 +644,33 @@ Glossary .. index:: single: generator generator - A function which returns a :term:`generator iterator`. It looks like a - normal function except that it contains :keyword:`yield` expressions - for producing a series of values usable in a for-loop or that can be - retrieved one at a time with the :func:`next` function. + Informally used to mean either a :term:`generator function` or a + :term:`generator iterator`, depending on context. The formal terms + :term:`generator function` and :term:`generator iterator` are uncommon + in practice; "generator" alone is almost always sufficient. + + .. index:: single: generator function - Usually refers to a generator function, but may refer to a - *generator iterator* in some contexts. In cases where the intended - meaning isn't clear, using the full terms avoids ambiguity. + generator function + A function which returns a :term:`generator` object. It looks like a + normal function except that it contains :keyword:`yield` expressions + for producing a series of values usable in a :keyword:`for`\-loop or + that can be retrieved one at a time with the :func:`next` function. + See :ref:`yieldexpr`. generator iterator - An object created by a :term:`generator` function. + An object created by a :term:`generator function` or a + :term:`generator expression`. Each :keyword:`yield` temporarily suspends processing, remembering the - execution state (including local variables and pending - try-statements). When the *generator iterator* resumes, it picks up where - it left off (in contrast to functions which start fresh on every - invocation). + execution state (including local variables and pending try-statements). + When the *generator iterator* resumes, it picks up where it left off + (in contrast to functions which start fresh on every invocation). + + Generator iterators also implement the :meth:`~generator.send` method + to send a value into the suspended generator, and the + :meth:`~generator.throw` method to raise an exception at the point + where the generator was paused. See :ref:`generator-methods`. .. index:: single: generator expression @@ -676,7 +689,7 @@ Glossary determined by the dispatch algorithm. See also the :term:`single dispatch` glossary entry, the - :func:`functools.singledispatch` decorator, and :pep:`443`. + :deco:`functools.singledispatch` decorator, and :pep:`443`. generic type A :term:`type` that can be parameterized; typically a diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst index 3adfedbf410ecc8..7a7a87cb9584001 100644 --- a/Doc/howto/a-conceptual-overview-of-asyncio.rst +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -115,7 +115,7 @@ The terms "coroutine function" and "coroutine object" are often conflated as coroutine. That can be confusing! In this article, coroutine specifically refers to a coroutine object, or more -precisely, an instance of :data:`types.CoroutineType` (native coroutine). +precisely, an instance of :class:`types.CoroutineType` (native coroutine). Note that coroutines can also exist as instances of :class:`collections.abc.Coroutine` -- a distinction that matters for type checking. diff --git a/Doc/howto/abi3t-migration.rst b/Doc/howto/abi3t-migration.rst new file mode 100644 index 000000000000000..c542efbdea8decc --- /dev/null +++ b/Doc/howto/abi3t-migration.rst @@ -0,0 +1,714 @@ +.. highlight:: c + +.. _abi3t-migration-howto: + +****************************************************** +Migrating to Stable ABI for free threading (``abi3t``) +****************************************************** + +Starting with the 3.15 release, CPython supports a variant of the Stable ABI +that supports :term:`free-threaded ` Python: +Stable ABI for Free-Threaded Builds, or ``abi3t`` for short. +This document describes how to adapt C API extensions to support free threading. + +Why do this +=========== + +The typical reason to use Stable ABI is to reduce the number of artifacts that +you need to build and distribute for each version of your library. + +Without the Stable ABI, you must build a separate shared library, and typically +a *wheel* distribution, for each feature version of CPython you wish +to support. +For example, each tag in the following table represents a separate +library/wheel: + ++-----------------+-----------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+=======================+========================+ +| 3.12 | ``cpython-312`` | --- | ++-----------------+-----------------------+------------------------+ +| 3.13 | ``cpython-313`` | ``cpython-313t`` | ++-----------------+-----------------------+------------------------+ +| 3.14 | ``cpython-314`` | ``cpython-314t`` | ++-----------------+-----------------------+------------------------+ +| 3.15 | ``cpython-315`` | ``cpython-315t`` | ++-----------------+-----------------------+------------------------+ +| 3.16 | ``cpython-316`` | ``cpython-316t`` | ++-----------------+-----------------------+------------------------+ +| Later versions | :samp:`cpython-3{XX}` | :samp:`cpython-3{XX}t` | ++-----------------+-----------------------+------------------------+ + +That's a lot of builds, especially when multiplied by the number +of supported platforms. + +With the Stable ABI (``abi3``, introduced in CPython 3.2), a single extension +(per platform) can cover all *non-free-threaded* builds of CPython: + ++-----------------+-------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+========================+ +| 3.12 | ``abi3`` | --- | ++-----------------+ +------------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+ +------------------------+ +| 3.15 | | ``cpython-315t`` | ++-----------------+ +------------------------+ +| 3.16 | | ``cpython-316t`` | ++-----------------+ +------------------------+ +| Later versions | | :samp:`cpython-3{XX}t` | ++-----------------+-------------------+------------------------+ + +The Stable ABI for free-threaded builds (``abi3t``), introduced in +CPython 3.15, does the same for free-threaded builds. +And it's compatible with non-free-threaded ones as well: + ++-----------------+-------------------+------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+==================+ +| 3.12 | ``abi3`` * | --- | ++-----------------+ +------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+-------------------+------------------+ +| 3.15 | ``abi3t`` | ++-----------------+ + +| 3.16 | | ++-----------------+ + +| Later versions | | ++-----------------+-------------------+------------------+ + +\* (As above, the ``abi3`` extension is compatible with all non-free-threaded +builds; even the 3.15+ ones that this table "attributes" to ``abi3t``.) + +Why *not* do this +----------------- + +There are two main downsides to Stable ABI. + +First, you extension may become slower, since Stable ABI prioritizes +compatibility over performance. +The difference is usually not noticeable, and often can be mitigated by +using the same source to build both a Stable ABI build and a few +version-specific ones for "tier 1" CPython versions. + +Second, not all of the C API is available. +Extensions need to be ported to build for Stable ABI, which may be difficult +or, in rare cases, impossible. + +Specifically, ``abi3t`` requires APIs added in CPython 3.15. +If you want to build your extension for older versions of CPython from the +same source, you have two main options: + +- Use preprocessor conditionals. + + When following this guide, use ``#ifdef Py_TARGET_ABI3T`` blocks whenever + you are told to do a change that breaks the build on CPython versions you + care about. Keep the pre-existing code in ``#else`` blocks. + + For hand-written C extensions, this approach is reasonable down to + CPython 3.12, due to additions introduced in :pep:`697`. + Keeping compatibility with 3.11 and below may be worth it for code + generators (for example, Cython). + +- Do not port to ``abi3t``, and continue building separate extensions for + each version of CPython, until you can drop support for the older versions. + + This is a valid approach. Not all extensions need to switch to ``abi3t`` + right now. + + +Prerequisites +============= + +This guide assumes that you have an extension written directly in C (or C++), +which you want to port to ``abi3t``. + +If your extenstion uses a code generator (like Cython) or language binding +(like PyO3), it's best to wait until that tool has support for ``abi3t``. +If you maintain such a tool, you might be able to adapt the instructions +here for your tool. + +Non-free-threaded Stable ABI +---------------------------- + +Your extension should support the Stable ABI (``abi3t``). +If not, either port it first, or follow this guide but be prepared to fix +issues it does not mention. + +Free-threading support +---------------------- + +While it's technically not a hard prerequisite, you will most likely want to +prepare your extension for free threading before you port it to ``abi3t``. +See :ref:`freethreading-extensions-howto` for instructions. + +.. seealso:: + + `Porting Extension Modules to Support Free-Threading + `__: + A community-maintained porting guide for extension authors. + +Isolating extension modules +--------------------------- + +Your module should use :ref:`multi-phase initialization `, +and it should either be isolated or limit itself to be loaded at most once +per process. +If it is not your case, follow :ref:`isolating-extensions-howto` first. +(See the :ref:`opt-out section ` for a shortcut.) + +Avoiding variable-sized types +----------------------------- + +If your extension defines variable-sized types (using :c:macro:`Py_tp_itemsize` +or :c:member:`PyTypeObject.tp_itemsize`), it cannot be ported to +``abi3t`` 3.15. + + +Setting up the build +==================== + +If you use a build tool (such as setuptools, meson-python, scikit-build-core), +search its documentation for a way to select ``abi3t``. +At the time of writing, not all of them have this; but if your tool does, +use it. +You may want to verify that it set the right flag by temporarily adding the +following just after ``#include ``:: + + #if Py_TARGET_ABI3T+0 <= 0x30f0000 + #error "abi3t define is not set!" + #endif + +This should result in a different error than "``abt3t`` define is not set". + +.. note:: + + If your build tool doesn't support ``abi3t`` yet, set the following macro + before including ``Python.h``:: + + #define Py_TARGET_ABI3T 0x30f0000 + + or specify it as a compiler flag, for example:: + + -DPy_TARGET_ABI3T=0x30f0000 + + Once your extension builds with this setting, it will be compatible with + CPython 3.15 and above. + + If you set this macro manually, you will later need to name and tag the + resulting extension manually as well. + This is covered in :ref:`abi3t-migration-tagging` below. + +This guide will ask you to make a series of changes. +After each one, verify that your extension still builds in the original +(non-``abi3t``) configuration, and ideally run tests on all Python +versions you support. +This will ensure that nothing breaks as you are porting. + + +.. _abi3t-howto-modexport: + +Module export hook +================== + +Unless you've done this step already, your extension module defines a +:ref:`module initialization function ` +named :samp:`PyInit_{}`. +You will need to port it to a :ref:`module export hook `, +:samp:`PyModExport_{}`, a feature added in CPython 3.15 in +:pep:`793`. + +Your existing init function should look like this (with your own names +for ```` and ````): + +.. code-block:: + :class: bad + + PyMODINIT_FUNC + PyInit_(void) + { + return PyModuleDef_Init(&); + } + +If there is some code before the ``return``, move it to +a :c:macro:`Py_mod_create` or :c:macro:`Py_mod_exec` slot function. +See the :ref:`PyInit documentation ` for related information. + +The function references a ``PyModuleDef`` object (```` in the code +above). +Its definition should be similar to the following, with different values +and perhaps some fields unnnamed or left out: + +.. code-block:: + :class: bad + + static PyModuleDef = { + PyModuleDef_HEAD_INIT, + .m_name = "my_module", + .m_doc = "my docstring", + .m_size = sizeof(my_state_struct), + .m_methods = my_methods, + .m_slots = my_slots, + .m_traverse = my_traverse, + .m_clear = my_clear, + .m_free = my_free, + }; + +Remove this definition and the ``PyInit`` function (or put them in +an ``#ifndef Py_TARGET_ABI3T`` block, to retain backwards compatibility), +and replace them with the following: + +.. code-block:: + :class: good + + PyABIInfo_VAR(abi_info); + + static PySlot my_slot_array[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "my_module"), + PySlot_STATIC_DATA(Py_mod_doc, "my docstring"), + PySlot_SIZE(Py_mod_state_size, sizeof(my_state_struct)), + PySlot_STATIC_DATA(Py_mod_methods, my_methods), + PySlot_STATIC_DATA(Py_mod_slots, my_slots), + PySlot_FUNC(Py_mod_state_traverse, my_traverse), + PySlot_FUNC(Py_mod_state_clear, my_clear), + PySlot_FUNC(Py_mod_state_free, my_free), + PySlot_END + }; + + PyMODEXPORT_FUNC + PyModExport_(void) + { + return my_slot_array; + } + +Leave out any fields that were missing (except the new :c:macro:`Py_mod_abi`), +and substitute your own values. + +See the :c:type:`PySlot` and :c:ref:`export hook ` +documentation for details on this API. + +As in the example, your ``PyModExport_`` function should *only* return a +pointer to static data. +If you cannot avoid additional code, refer to the +:ref:`caveats in PyModExport documentation `. + + +Existing slots +-------------- + +If you have a ``Py_mod_slots`` slot, check the array it refers to. +It should be a :c:type:`PyModuleDef_Slot` array like the following: + +.. code-block:: + :class: bad + + static PyObject *create_module(PyObject *spec, PyModuleDef *def) { ... } + static int my_first_module_exec(PyObject *module) { ... } + static int my_second_module_exec(PyObject *module) { ... } + + static PyModuleDef_Slot my_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_create, my_module_create}, + {Py_mod_exec, my_first_module_exec}, + {Py_mod_exec, my_second_module_exec}, + {0, NULL} + }; + +``py_mod_create`` +................. + + +If you have a :c:macro:`Py_mod_create` entry, make sure the function can be +called with ``NULL`` as its second argument (instead of the +:c:type:`PyModuleDef`, which you are removing). +Often, this argument isn't used at all; you can check by renaming it: + +.. code-block:: + :class: good + + static PyObject *create_module(PyObject *spec, PyModuleDef *_unused) { ... } + +If the argument is used, find a different way to pass in the data. +Commonly, the information is static and you can refer to it directly. +(If you're reusing a single function for several different modules, consider +defining several functions instead.) + + +Multiple ``py_mod_exec`` +........................ + +If you have *more than one* :c:macro:`Py_mod_exec` entry, consolidate them: +create a new function that calls the others, and replace existing slots +with it. + +.. code-block:: + :class: good + + static int my_module_exec(PyObject *module) { + if (my_first_module_exec(module) < 0) return -1; + if (my_second_module_exec(module) < 0) return -1; + } + + static PyModuleDef_Slot my_slots[] = { + ... + /* (remove other Py_mod_exec slots) */ + ... + {Py_mod_exec, my_module_exec}, + {0, NULL} + }; + +If the functions aren't used elsewhere, you can combine their bodies instead. + + +Merging slot arrays +................... + +Optionally, when you break compatibility with Python 3.14, you may clean up +the code by moving slots into the :c:type:`PySlot` array, and converting the +definitions to :c:macro:`PySlot_DATA` and :c:macro:`PySlot_FUNC`: + +.. code-block:: + :class: good + + static PySlot my_slot_array[] = { + ... + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_DATA(Py_mod_multiple_interpreters, + Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) + PySlot_FUNC(Py_mod_create, my_module_create), + PySlot_FUNC(Py_mod_exec, my_module_exec), + PySlot_END + }; + +If you do this, delete the original :c:type:`PyModuleDef_Slot` array and +its ``Py_mod_slots`` entry. + + +Associated ``PyModuleDef`` +-------------------------- + +Since the new API does not use a :c:type:`!PyModuleDef` structure, a definition +will not be associated with the resulting module. +This changes the behavior of the following functions: + +- :c:func:`PyModule_GetDef` +- :c:func:`PyType_GetModuleByDef` + +Check your code for these. +If you do not use them, you can skip this section. + +These functions are typically used for two purposes: + +1. To get the definition the module was created with. + This is no longer possible using the new API. + Modules no longer keep a reference to the definition, so you will need to + figure out a different way to pass the relevant data around. + +.. _abi3t-migration-module-token: + +2. To check if a given module object is “yours”. + This use case is now served by :ref:`module tokens ` -- + opaque pointers that identify a module. + To use a token, declare (or reuse) a unique static variable, for example: + + .. code-block:: + :class: good + + static char my_token; + + and add a pointer to it in a new entry to your module's ``PySlot`` array: + + .. code-block:: + :class: good + :emphasize-lines: 3 + + static PySlot my_slot_array[] = { + ... + PySlot_STATIC_DATA(Py_mod_token, &my_token), + PySlot_END + } + + Then, switch from :c:func:`PyModule_GetDef` calls such as: + + .. code-block:: + :class: bad + + PyModuleDef *def = PyModule_GetDef(module); + + to :c:func:`PyModule_GetToken` (which uses an output argument and may fail + with an exception): + + .. code-block:: + :class: good + + void *token; + if (PyModule_GetToken(module, &token) < 0) { + /* handle error */ + } + + and from :c:func:`PyType_GetModuleByDef` calls such as: + + .. code-block:: + :class: bad + + PyObject *module = PyType_GetModuleByDef(type, my_def); + /* handle error; use module */ + + to :c:func:`PyType_GetModuleByToken` (which returns a strong reference): + + .. code-block:: + :class: good + + PyObject *module = PyType_GetModuleByToken(type, my_token); + /* handle error; use module */ + Py_XDECREF(module); + +``PyObject`` opaqueness +======================= + +The :c:type:`PyObject` and :c:type:`PyVarObject` structures are opaque +in ``abi3t``. + +Accessing their members is prohibited. +If you do this, switch to getter/setter functions mentioned in +their documentation: + +- :c:member:`PyObject.ob_type` +- :c:member:`PyObject.ob_refcnt` +- :c:member:`PyVarObject.ob_size` + +Also, the *size* of the :c:type:`PyObject` structures is +unknown to the compiler. +It can -- and *does* -- change between different CPython builds. + +.. note:: + + While the size is available at runtime (for example as + ``sys.getsizeof(object())`` in Python code), you should resist the + temptation to calculate pointer offsets from it. + The object memory layout is subject to change in future + ``abi3t`` implementations. + + +Custom type definitions +----------------------- + +Since :c:type:`!PyObject` is opaque, the traditional way of defining +custom types no longer works: + +.. code-block:: + :class: bad + + typedef struct { + PyObject_HEAD // expands to `PyObject ob_base;` which has unknown size + + int my_data; + } CustomObject; + + static PyType_Spec CustomType_spec = { + ... + .basicsize = sizeof(CustomObject), + ... + }; + +Most likely, all your class definitions, *and* all code that accesses +your classes' data, will need to be rewritten. +This will probably be the biggest change you need to support ``abi3t``. + +For each such type, instead of defining a ``struct`` for the entire instance, +define one with only the “additional” fields -- ones specific to your class, +not its superclasses: + +.. code-block:: + :class: good + + typedef struct { + int my_data; + } CustomObjectData; + +Change the name. +Almost all code that uses the struct will need to change +(notably, pointers to the new structure cannot be cast to/from ``PyObject*``), +and changing the name will highlight the usages as compiler errors. +(If you use ``typeof``, C++ ``auto``, or similar ways to avoid +typing the type name, this won't work. Be extra careful, and consider running +tools to detect undefined behavior.) + +Then, to create the class, use *negative* ``basicsize`` to indicate +“extra” storage space rather than *total* instance size: + +.. code-block:: + :class: good + + static PyType_Spec CustomType_spec = { + ... + .basicsize = -sizeof(CustomObjectData), /* note the minus sign */ + ... + }; + +If you use :c:macro:`Py_tp_members`, set the :c:macro:`Py_RELATIVE_OFFSET` +flag on each member and specify the :c:member:`~PyMemberDef.offset` +relative to your new struct. + + +Custom type data access +----------------------- + +Then comes the hard part: in all code that needs to access this struct, +you will need an additional :c:func:`PyObject_GetTypeData` call to +retrieve a ``CustomObjectData *`` pointer from ``PyObject *``: + +.. code-block:: + :class: good + + PyObject *obj = ...; + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + +Note that this call requires the *type object* for your class (``cls``). + +If your class is not subclassable (that is, it does not use the +:c:macro:`Py_TPFLAGS_BASETYPE` flag), ``cls`` will be ``Py_TYPE(obj)``. +Otherwise, **DO NOT USE** ``Py_TYPE`` with :c:func:`!PyObject_GetTypeData`: +it might return memory reserved to an unrelated subclass! +For example, if a user makes a subclass like this: + +.. code-block:: python + + class Sub(YourCustomClass): + __slots__ = ('a', 'b') + +then ``Py_TYPE(obj)`` is ``Sub``, and the underlying memory may +look like this: + +.. code-block:: text + + ╭─ PyObject *obj + │ ╭─ the pointer you want + │ │ ╭─ PyObject_GetTypeData(obj, Py_TYPE(obj)) + ▼ ▼ ▼ + ┌──────────┬───┬────────────────┬───┬─────────────┬───┬─────────────┐ + │ PyObject │...│ CustomTypeData │...│ PyObject *a │...│ PyObject *b │ + └──────────┴───┴────────────────┴───┴─────────────┴───┴─────────────┘ + +(Ellipses indicate possible padding. +Note that this memory layout is not guaranteed: future versions of Python may +add different padding or even switch the order of the structures.) + +There are two main ways to get the right class: + +- In instance methods, your implementation may use the :c:type:`PyCMethod` + signature (and the :c:macro:`METH_METHOD` bit in + :c:member:`PyMethodDef.ml_flags`), + and get the class as the ``defining_class`` argument. +- Otherwise, give your class a unique static token using the + :c:macro:`Py_tp_token` slot, and use: + + .. code-block:: + :class: good + + PyTypeObject cls; + if (PyType_GetBaseByToken(Py_TYPE(obj), my_tp_token, &cls) < 0) { + /* handle error */ + } + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + + Type tokens work similarly to module tokens covered :ref:`earlier in this + guide `. + + + +Avoid build-time conditionals +============================= + +Check your code for API that identifies the version of Python used to +*build* your extension. +This no longer corresponds to the Python your extension runs on, so code +that uses this information often needs changing. +The macros to check for are: + +- :c:macro:`PY_VERSION_HEX`, :c:macro:`PY_MAJOR_VERSION`, + :c:macro:`PY_MINOR_VERSION`: + + - to get the run-time version, use :c:data:`Py_Version`; + - to determine what C API is available, use :c:macro:`Py_TARGET_ABI3T`. + This macro is set to the minimum supported version. + +- :c:macro:`Py_GIL_DISABLED`: under ``abi3t``, this macro is always defined. + Code that works with free-threaded Python *should* also work with + the GIL enabled (since the GIL can be enabled at run time), + and usually *does* (unless it, for some reason, requires more than one + :term:`attached thread state ` at one time). + + +Further code changes +==================== + +If you are still left with compiler errors or warnings, find a way to fix them. +Alas, this guide is limited, and cannot cover all possible code +changes extensions may need. + +If you find a problem that other extension authors might run into, +consider :ref:`reporting an issue ` (or sending +a pull request) for this guide. + +It is possible your issue cannot be fixed for the current version of ``abi3t``. +In that case, reporting it may help it get prioritized for the next version +of CPython. + + +.. _abi3t-migration-tagging: + +Tagging and distribution +======================== + +If you are using a build tool with ``abi3t`` support, your extension is ready, +but you might want to check that it was built correctly. + +Extensions built with ``abi3t`` should have the following extension: + +- On Windows: ``.pyd`` (like any other extension); +- Linux, macOS, and other systems that use the ``.so`` suffix: ``.abi3t.so`` + (**not** ``.cpython-315t.so`` or ``.abi3.so``). + Note that both free-threaded and non-free-threaded builds will + load ``.abi3t.so`` extensions; +- Other systems: consult your distributor, and perhaps update this guide. + +If you distribute the extension as a *wheel*, use the following tags: + +* Python tag: :samp:`cp3{XX}`, where *XX* is the minimum Python version + the extension is built for. + (For example, ``cp315`` if you set ``Py_TARGET_ABI3T`` to ``0x30f0000``. + See :ref:`abi3-compiling` for more values.) +* ABI tag: ``abi3.abi3t``. This is a *compressed tag set* that indicates + support for both non-free-threaded and free-threaded builds. + +For example, the wheel filename may look like this: + +.. code-block:: text + + myproject-1.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl + +.. seealso:: `Platform Compatibility Tags `__ in the PyPA package distribution metadata. + +If the filename or tags are incorrect, fix them. + + +Testing +======= + +Note that when you build an extension compatible with multiple versions of +CPython, you should always *test* it with each version it supports (for +example, 3.15, 3.16, and so on). +Stable ABI only guarantees *ABI* compatibility; there may also be behavior +changes -- both intentional ones (covered by :pep:`387`) and bugs. + +Be sure to run tests on both free-threaded and non-free-threaded builds +of CPython. + +If they pass, congratulations! You have an ``abi3t`` extension. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index d7deb6c6bc1768f..fcc4fe838c783ba 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -4,8 +4,6 @@ Annotations Best Practices ************************** -:author: Larry Hastings - .. topic:: Abstract This document is designed to encapsulate the best practices @@ -154,7 +152,7 @@ on an arbitrary object ``o``: as the ``globals``, and ``dict(vars(o))`` as the ``locals``, when calling :func:`eval`. * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively + :deco:`functools.wraps`, or :func:`functools.partial`, iteratively unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as appropriate, until you have found the root unwrapped function. * If ``o`` is a callable (but not a class), use diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 902c50de00803c5..0cb8c5cc3ebd369 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -4,8 +4,6 @@ Argparse Tutorial ***************** -:author: Tshepang Mbambo - .. currentmodule:: argparse This tutorial is intended to be a gentle introduction to :mod:`argparse`, the diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 816639552d7cd6d..e5f85e0110321c8 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -6,9 +6,6 @@ .. currentmodule:: curses -:Author: A.M. Kuchling, Eric S. Raymond -:Release: 2.04 - .. topic:: Abstract diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index a7a68281860cb5d..7233898860ada14 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -4,9 +4,6 @@ Descriptor Guide ================ -:Author: Raymond Hettinger -:Contact: - .. Contents:: @@ -28,7 +25,7 @@ This guide has four major sections: 4) The last section has pure Python equivalents for built-in descriptors that are written in C. Read this if you're curious about how functions turn into bound methods or about the implementation of common tools like - :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :deco:`classmethod`, :deco:`staticmethod`, :deco:`property`, and :term:`__slots__`. @@ -317,8 +314,8 @@ Descriptors invert that relationship and allow the data being looked-up to have a say in the matter. Descriptors are used throughout the language. It is how functions turn into -bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, -:func:`property`, and :func:`functools.cached_property` are all implemented as +bound methods. Common tools like :deco:`classmethod`, :deco:`staticmethod`, +:deco:`property`, and :deco:`functools.cached_property` are all implemented as descriptors. @@ -1326,7 +1323,7 @@ example calls are unexciting: 30 Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this: +:deco:`staticmethod` would look like this: .. testcode:: @@ -1466,7 +1463,7 @@ Now a new dictionary of unique keys can be constructed like this: {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this: +:deco:`classmethod` would look like this: .. testcode:: @@ -1604,7 +1601,7 @@ matters when a large number of instances are going to be created. 4. Improves speed. Reading instance variables is 35% faster with ``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). -5. Blocks tools like :func:`functools.cached_property` which require an +5. Blocks tools like :deco:`functools.cached_property` which require an instance dictionary to function correctly: .. testcode:: diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 2fe5814bb04a736..adb9dc9a4879eb5 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -256,7 +256,7 @@ Ensuring unique enumeration values ---------------------------------- By default, enumerations allow multiple names as aliases for the same value. -When this behavior isn't desired, you can use the :func:`unique` decorator:: +When this behavior isn't desired, you can use the :deco:`unique` decorator:: >>> from enum import Enum, unique >>> @unique @@ -509,7 +509,7 @@ to use the standard :func:`repr`. .. note:: - Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum` + Adding :deco:`~dataclasses.dataclass` decorator to :class:`Enum` and its subclasses is not supported. It will not raise any errors, but it will produce very strange results at runtime, such as members being equal to each other:: diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index b21ed1c8f37be11..ad0578df0a27029 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -218,13 +218,15 @@ Thread State and GIL APIs Python provides a set of functions and macros to manage thread state and the GIL, such as: +* :c:func:`PyThreadState_Ensure`, :c:func:`PyThreadState_EnsureFromView`, + and :c:func:`PyThreadState_Release` * :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread` * :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` These functions should still be used in the free-threaded build to manage thread state even when the :term:`GIL` is disabled. For example, if you -create a thread outside of Python, you must call :c:func:`PyGILState_Ensure` +create a thread outside of Python, you must call :c:func:`PyThreadState_Ensure` before calling into the Python API to ensure that the thread has a valid Python thread state. diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 380c2be04957d5e..53bea1db191d76f 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -165,3 +165,132 @@ to false. If the flag is true then the :class:`warnings.catch_warnings` context manager uses a context variable for warning filters. If the flag is false then :class:`~warnings.catch_warnings` modifies the global filters list, which is not thread-safe. See the :mod:`warnings` module for more details. + + +Increased memory usage +---------------------- + +The free-threaded build will typically use more memory compared to the default +build. There are multiple reasons for this, mostly due to design decisions. + + +All interned strings are immortal +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For modern Python versions (since version 2.3), interning a string (e.g. with +:func:`sys.intern`) does not cause it to become immortal. Instead, if the last +reference to that string disappears, it will be removed from the interned +string table. This is not the case for the free-threaded build and any interned +string will become immortal, surviving until interpreter shutdown. + + +Non-GC objects have a larger object header +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The free-threaded build uses a different :c:type:`PyObject` structure. Instead +of having the GC related information allocated before the :c:type:`PyObject` +structure, like in the default build, the GC related info is part of the normal +object header. For example, on the AMD64 platform, ``None`` uses 32 bytes on +the free-threaded build vs 16 bytes for the default build. GC objects (such as +dicts and lists) are the same size for both builds since the free-threaded +build does not use additional space for the GC info. + + +QSBR can delay freeing of memory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In order to safely implement lock-free data structures, a safe memory +reclamation (SMR) scheme is used, known as quiescent state-based reclamation +(QSBR). This means that the memory backing data structures allowing lock-free +access will use QSBR, which defers the free operation, rather than immediately +freeing the memory. Two examples of these data structures are the list object +and the dictionary keys object. See ``InternalDocs/qsbr.md`` in the CPython +source tree for more details on how QSBR is implemented. Running +:func:`gc.collect` should cause all memory being held by QSBR to be actually +freed. Note that even when QSBR frees the memory, the underlying memory +allocator may not immediately return that memory to the OS and so the resident +set size (RSS) of the process might not decrease. + + +mimalloc allocator vs pymalloc +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default build will normally use the "pymalloc" memory allocator for small +allocations (512 bytes or smaller). The free-threaded build does not use +pymalloc and allocates all Python objects using the "mimalloc" allocator. The +pymalloc allocator has the following properties that help keep memory usage +low: small per-allocated-block overhead, effective memory fragmentation +prevention, and quick return of free memory to the operating system. The +mimalloc allocator does quite well in these respects as well but can have some +more overhead. + +In the free-threaded build, mimalloc manages memory in a number of separate +heaps (currently four). For example, all GC supporting objects are allocated +from their own heap. Using separate heaps means that free memory in one heap +cannot be used for an allocation that uses another heap. Also, some heaps are +configured to use QSBR (quiescent-state based reclamation) when freeing the +memory that backs up the heap (known as "pages" in mimalloc terminology). The +use of QSBR creates a delay between all memory blocks for a page being freed +and the memory page being released, either for new allocations or back to the +OS. + +The mimalloc allocator also defers returning freed memory back to the OS. You +can reduce that delay by setting the environment variable +:envvar:`!MIMALLOC_PURGE_DELAY` to ``0``. Note that this will likely reduce +the performance of the allocator. + + +Free-threaded reference counting can cause objects to live longer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the default build, when an object's reference count reaches zero, it is +normally deallocated. The free-threaded build uses "biased reference +counting", with a fast-path for objects "owned" by the current thread and a +slow path for other objects. See :pep:`703` for additional details. Any time +an object's reference count ends up in a "queued" state, deallocation can be +deferred. The queued state is cleared from the "eval breaker" section of the +bytecode evaluator. + +The free-threaded build also allows a different mode of reference counting, +known as "deferred reference counting". This mode is enabled by setting a flag +on a per-object basis. Deferred reference counting is enabled for the +following types: + +* module objects +* module top-level functions +* class methods defined in the class scope +* descriptor objects +* thread-local objects, created by :class:`threading.local` + +When deferred reference counting is enabled, references from Python function +stacks are not added to the reference count. This scheme reduces the overhead +of reference counting, especially for objects used from multiple threads. +Because the stack references are not counted, objects with deferred reference +counting are not immediately freed when their internal reference count goes to +zero. Instead, they are examined by the next GC run and, if no stack +references to them are found, they are freed. This means these objects are +freed by the GC and not when their reference count goes to zero, as is typical. + + +Per-thread reference counting can delay freeing objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To avoid contention on the reference count fields of frequently shared +objects, the free-threaded build also uses "per-thread reference counting" +for a few selected object types. Rather than updating a single shared +reference count, each thread maintains its own local reference count array, +indexed by a unique id assigned to the object. The true reference count is +only computed by summing the per-thread counts when the object's local +count drops to zero. Per-thread reference counting is currently used for: + +* heap type objects (classes created in Python) +* code objects +* the ``__dict__`` of module objects + +Because the per-thread counts must be merged back to the object before it +can be deallocated, objects using per-thread reference counting are +typically freed later than they would be in the default build. In +particular, such an object is usually not freed until the thread that +referenced it reaches a safe point (for example, in the "eval breaker" +section of the bytecode evaluator) or exits. Running :func:`gc.collect` +will merge the per-thread counts and allow these objects to be freed. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 552514063c95ab2..a61fdaee27f6b18 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -4,9 +4,6 @@ Functional Programming HOWTO ******************************** -:Author: \A. M. Kuchling -:Release: 0.32 - In this document, we'll take a tour of Python's features suitable for implementing programs in a functional style. After an introduction to the concepts of functional programming, we'll look at language features such as @@ -1042,7 +1039,7 @@ first calculation. :: >>> functools.reduce(operator.concat, []) Traceback (most recent call last): ... - TypeError: reduce() of empty sequence with no initial value + TypeError: reduce() of empty iterable with no initial value >>> functools.reduce(operator.mul, [1, 2, 3], 1) 6 >>> functools.reduce(operator.mul, [], 1) @@ -1185,7 +1182,8 @@ about whether this lambda-free style is better. Revision History and Acknowledgements ===================================== -The author would like to thank the following people for offering suggestions, +This HOWTO was originally written by A. M. Kuchling. The author would like to +thank the following people for offering suggestions, corrections and assistance with various drafts of this article: Ian Bicking, Nick Coghlan, Nick Efford, Raymond Hettinger, Jim Jewett, Mike Krell, Leandro Lameiro, Jussi Salmela, Collin Winter, Blake Winton. @@ -1239,9 +1237,9 @@ Text Processing". Mertz also wrote a 3-part series of articles on functional programming for IBM's DeveloperWorks site; see -`part 1 `__, -`part 2 `__, and -`part 3 `__, +`part 1 `__, +`part 2 `__, and +`part 3 `__. Python documentation diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 81fc7e63f35bd79..57e2d6e0752447f 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -37,6 +37,7 @@ Python Library Reference. mro.rst free-threading-python.rst free-threading-extensions.rst + abi3t-migration.rst remote_debugging.rst General: @@ -61,6 +62,7 @@ Advanced development: * :ref:`freethreading-python-howto` * :ref:`freethreading-extensions-howto` * :ref:`isolating-extensions-howto` +* :ref:`abi3t-migration-howto` * :ref:`python_2.3_mro` * :ref:`socket-howto` * :ref:`timerfd-howto` diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 06c1ae40da5e67e..8f0b0c41ea48617 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -6,9 +6,6 @@ Instrumenting CPython with DTrace and SystemTap =============================================== -:author: David Malcolm -:author: Łukasz Langa - DTrace and SystemTap are monitoring tools, each providing a way to inspect what the processes on a computer system are doing. They both use domain-specific languages allowing a user to write scripts which: diff --git a/Doc/howto/ipaddress.rst b/Doc/howto/ipaddress.rst index e852db98156facc..646c4c4d9e7ab14 100644 --- a/Doc/howto/ipaddress.rst +++ b/Doc/howto/ipaddress.rst @@ -8,9 +8,6 @@ An introduction to the ipaddress module *************************************** -:author: Peter Moody -:author: Nick Coghlan - .. topic:: Overview This document aims to provide a gentle introduction to the diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 0ee4c0086dd98ce..87025814aafb9ab 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4,8 +4,6 @@ Logging Cookbook ================ -:Author: Vinay Sajip - This page contains a number of recipes related to logging, which have been found useful in the past. For links to tutorial and reference information, please see :ref:`cookbook-ref-links`. diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 454e2f4930e724d..c8ce0df9e937f8c 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -4,8 +4,6 @@ Logging HOWTO ============= -:Author: Vinay Sajip - .. _logging-basic-tutorial: .. currentmodule:: logging diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 653f28ddbabfa4b..5565f99b244f11b 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -6,8 +6,6 @@ Python support for the ``perf map`` compatible profilers ======================================================== -:author: Pablo Galindo - `The Linux perf profiler `_ and `samply `_ are powerful tools that allow you to profile and obtain information about the performance of your application. @@ -218,8 +216,8 @@ How to obtain the best results ------------------------------ For best results, keep frame pointers enabled. On supported GCC-compatible -toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and, when -available, ``-mno-omit-leaf-frame-pointer`` by default. These flags allow +toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and similar +flags (see :option:`--without-frame-pointers` for details). These flags allow profilers to unwind using only the frame pointer and not on DWARF debug information. This is because as the code that is interposed to allow ``perf`` support is dynamically generated it doesn't have any DWARF debugging information diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 9f73c811cfcbc0d..f19f6006dbac9e7 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -6,8 +6,6 @@ How to port Python 2 Code to Python 3 ************************************* -:author: Brett Cannon - Python 2 reached its official end-of-life at the start of 2020. This means that no new bug reports, fixes, or changes will be made to Python 2 - it's no longer supported: see :pep:`373` and diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 6fc087c3f1c3673..ecdb35136a99f3b 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -4,8 +4,6 @@ Regular expression HOWTO **************************** -:Author: A.M. Kuchling - .. TODO: Document lookbehind assertions Better way of displaying a RE, a string, and what it matches diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index cbc49d15a0771b9..b17ab3f4391dad0 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -4,9 +4,6 @@ Socket Programming HOWTO **************************** -:Author: Gordon McMillan - - .. topic:: Abstract Sockets are used nearly everywhere, but are one of the most severely diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 70c34cde8a06592..b511aa4e59fe1e7 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -3,9 +3,6 @@ Sorting Techniques ****************** -:Author: Andrew Dalke and Raymond Hettinger - - Python lists have a built-in :meth:`list.sort` method that modifies the list in-place. There is also a :func:`sorted` built-in function that builds a new sorted list from an iterable. @@ -375,7 +372,7 @@ Odds and Ends :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__` for details on the mechanics). To avoid surprises, :pep:`8` recommends that all six comparison methods be implemented. - The :func:`~functools.total_ordering` decorator is provided to make that + The :deco:`~functools.total_ordering` decorator is provided to make that task easier. * Key functions need not depend directly on the objects being sorted. A key diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 243cc27bac7025d..cbd53d5b619f5a3 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -4,8 +4,6 @@ Unicode HOWTO ***************** -:Release: 1.12 - This HOWTO discusses Python's support for the Unicode specification for representing textual data, and explains various problems that people commonly encounter when trying to work with Unicode. diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 4e77d2cb407f726..e4f218f088ba89d 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -4,9 +4,6 @@ HOWTO Fetch Internet Resources Using The urllib Package *********************************************************** -:Author: `Michael Foord `_ - - Introduction ============ diff --git a/Doc/includes/capi-extension/spammodule-01.c b/Doc/includes/capi-extension/spammodule-01.c index 0bc34ef57445cb9..8ddb416e7d6cd08 100644 --- a/Doc/includes/capi-extension/spammodule-01.c +++ b/Doc/includes/capi-extension/spammodule-01.c @@ -37,12 +37,12 @@ static PyMethodDef spam_methods[] = { PyABIInfo_VAR(abi_info); -static PyModuleDef_Slot spam_slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "spam"}, - {Py_mod_doc, "A wonderful module with an example function"}, - {Py_mod_methods, spam_methods}, - {0, NULL} +static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_STATIC_DATA(Py_mod_doc, "A wonderful module with an example function"), + PySlot_STATIC_DATA(Py_mod_methods, spam_methods), + PySlot_END }; /// Export hook prototype diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 47f5eabb6f2180f..13f463a1e95340e 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -36,11 +36,6 @@ This module defines the following constants and functions: This is now a synonym of the built-in :exc:`RuntimeError`. -.. data:: LockType - - This is the type of lock objects. - - .. function:: start_new_thread(function, args[, kwargs]) Start a new thread and return its identifier. The thread executes the @@ -162,58 +157,66 @@ This module defines the following constants and functions: .. versionadded:: 3.2 -Lock objects have the following methods: +.. raw:: html + + + + + +.. class:: LockType -.. method:: lock.acquire(blocking=True, timeout=-1) + This is the type of lock objects. - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). + Lock objects have the following methods: - If the *blocking* argument is present, the action depends on its - value: if it is false, the lock is only acquired if it can be acquired - immediately without waiting, while if it is true, the lock is acquired - unconditionally as above. + .. method:: acquire(blocking=True, timeout=-1) - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is false. + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. + If the *blocking* argument is present, the action depends on its + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired + unconditionally as above. - .. versionchanged:: 3.2 - The *timeout* parameter is new. + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *blocking* is false. - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. - .. versionchanged:: 3.14 - Lock acquires can now be interrupted by signals on Windows. + .. versionchanged:: 3.2 + The *timeout* parameter is new. + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. -.. method:: lock.release() + .. versionchanged:: 3.14 + Lock acquires can now be interrupted by signals on Windows. - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. + .. method:: release() + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. -.. method:: lock.locked() + .. method:: locked() - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: + In addition to these methods, lock objects can also be used via the + :keyword:`with` statement, e.g.:: - import _thread + import _thread - a_lock = _thread.allocate_lock() + a_lock = _thread.allocate_lock() - with a_lock: - print("a_lock is locked while this executes") + with a_lock: + print("a_lock is locked while this executes") **Caveats:** diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 8112cfee7d204db..be25a94e7e94f6f 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -166,17 +166,17 @@ The :mod:`!abc` module also provides the following decorator: or is derived from it. A class that has a metaclass derived from :class:`!ABCMeta` cannot be instantiated unless all of its abstract methods and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`!abstractmethod` may be used + of the normal 'super' call mechanisms. :deco:`!abstractmethod` may be used to declare abstract methods for properties and descriptors. Dynamically adding abstract methods to a class, or attempting to modify the abstraction status of a method or class once it is created, are only supported using the :func:`update_abstractmethods` function. The - :func:`!abstractmethod` only affects subclasses derived using regular + :deco:`!abstractmethod` only affects subclasses derived using regular inheritance; "virtual subclasses" registered with the ABC's :meth:`~ABCMeta.register` method are not affected. - When :func:`!abstractmethod` is applied in combination with other method + When :deco:`!abstractmethod` is applied in combination with other method descriptors, it should be applied as the innermost decorator, as shown in the following usage examples:: @@ -214,7 +214,7 @@ The :mod:`!abc` module also provides the following decorator: the descriptor must identify itself as abstract using :attr:`!__isabstractmethod__`. In general, this attribute should be ``True`` if any of the methods used to compose the descriptor are abstract. For - example, Python's built-in :class:`property` does the equivalent of:: + example, Python's built-in :deco:`property` does the equivalent of:: class Descriptor: ... @@ -237,14 +237,14 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractclassmethod .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`classmethod` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`classmethod` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`classmethod`, indicating an abstract - classmethod. Otherwise it is similar to :func:`abstractmethod`. + A subclass of the built-in :class:`classmethod`, indicating an abstract + classmethod. Otherwise it is similar to :deco:`abstractmethod`. - This special case is deprecated, as the :func:`classmethod` decorator + This special case is deprecated, as the :deco:`classmethod` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -258,14 +258,14 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractstaticmethod .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`staticmethod` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`staticmethod` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`staticmethod`, indicating an abstract - staticmethod. Otherwise it is similar to :func:`abstractmethod`. + A subclass of the built-in :class:`staticmethod`, indicating an abstract + staticmethod. Otherwise it is similar to :deco:`abstractmethod`. - This special case is deprecated, as the :func:`staticmethod` decorator + This special case is deprecated, as the :deco:`staticmethod` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -278,15 +278,15 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractproperty - .. deprecated:: 3.3 - It is now possible to use :class:`property`, :meth:`property.getter`, - :meth:`property.setter` and :meth:`property.deleter` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`property`, :deco:`property.getter`, + :deco:`property.setter` and :deco:`property.deleter` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`property`, indicating an abstract + A subclass of the built-in :class:`property`, indicating an abstract property. - This special case is deprecated, as the :func:`property` decorator + This special case is deprecated, as the :deco:`property` decorator is now correctly identified as abstract when applied to an abstract method:: diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index e37afd6d0b6d5aa..e4a5f4d109b4992 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -442,9 +442,8 @@ is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. .. note:: - Empty lines are treated as empty strings (``''``), which are allowed as values but - not as arguments. Empty lines that are read as arguments will result in an - "unrecognized arguments" error. + Each line is treated as a single argument, so an empty line is read as an + empty string (``''``). :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -637,25 +636,22 @@ are set. .. versionadded:: 3.14 -To highlight inline code in your description or epilog text, you can use -backticks:: +To highlight inline code in your description, epilog, or argument ``help`` +text, you can use single or double backticks:: >>> parser = argparse.ArgumentParser( ... formatter_class=argparse.RawDescriptionHelpFormatter, + ... description='Run ``python -m myapp`` to start.', ... epilog='''Examples: ... `python -m myapp --verbose` - ... `python -m myapp --config settings.json` + ... ``python -m myapp --config settings.json`` ... ''') + >>> parser.add_argument('--foo', help='set the `foo` value') When colors are enabled, the text inside backticks will be displayed in a distinct color to help examples stand out. When colors are disabled, backticks are preserved as-is, which is readable in plain text. -.. note:: - - Backtick markup only applies to description and epilog text. It does not - apply to individual argument ``help`` strings. - .. versionadded:: 3.15 @@ -1055,6 +1051,10 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +Because ``nargs='*'`` gathers any supplied values into a list, an absent +positional argument yields an empty list (``[]``). Only a non-``None`` +*default* overrides this (so ``default=None`` still gives ``[]``). + For required_ arguments, the ``default`` value is ignored. For example, this applies to positional arguments with nargs_ values other than ``?`` or ``*``, or optional arguments marked as ``required=True``. @@ -1372,6 +1372,11 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +Multiple arguments may share the same ``dest``. By default, the value from the +last such argument given on the command line wins. Use ``action='append'`` to +collect values from all of them into a list instead. For conflicting *option +strings* rather than ``dest`` names, see conflict_handler_. + .. versionchanged:: 3.15 Single-dash long option now takes precedence over short options. @@ -1780,6 +1785,11 @@ Subcommands present, and when the ``b`` command is specified, only the ``foo`` and ``baz`` attributes are present. + If a subparser defines an argument with the same ``dest`` as the parent + parser, the two share a single namespace attribute, so the parent's value + won't be retained. Users should give them distinct ``dest`` values to + keep both. + Similarly, when a help message is requested from a subparser, only the help for that particular parser will be printed. The help message will not include parent parser or sibling parser messages. (A help message for each @@ -2235,6 +2245,9 @@ Customizing file parsing def convert_arg_line_to_args(self, arg_line): return arg_line.split() + Note that with this override an argument can no longer contain spaces, since + each space-separated word becomes a separate argument. + Exiting methods ^^^^^^^^^^^^^^^ diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 3cd107460596d1b..c688d54318e707f 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -12,7 +12,7 @@ This module defines an object type which can compactly represent an array of basic values: characters, integers, floating-point numbers, complex numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a -:dfn:`type code`, which is a single character. The following type codes are +:dfn:`type code`. The following type codes are defined: +-----------+--------------------+-------------------+-----------------------+-------+ @@ -22,9 +22,7 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'B'`` | unsigned char | int | 1 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'u'`` | wchar_t | Unicode character | 2 | \(1) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(2) | +| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -42,39 +40,24 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'e'`` | _Float16 | float | 2 | \(3) | +| ``'e'`` | _Float16 | float | 2 | \(2) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'d'`` | double | float | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'F'`` | float complex | complex | 8 | \(4) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'D'`` | double complex | complex | 16 | \(4) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'Zf'`` | float complex | complex | 8 | \(4) | +| ``'Zf'`` | float complex | complex | 8 | \(3) | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'Zd'`` | double complex | complex | 16 | \(4) | +| ``'Zd'`` | double complex | complex | 16 | \(3) | +-----------+--------------------+-------------------+-----------------------+-------+ Notes: (1) - It can be 16 bits or 32 bits depending on the platform. - - .. versionchanged:: 3.9 - ``array('u')`` now uses :c:type:`wchar_t` as C type instead of deprecated - ``Py_UNICODE``. This change doesn't affect its behavior because - ``Py_UNICODE`` is alias of :c:type:`wchar_t` since Python 3.3. - - .. deprecated-removed:: 3.3 3.16 - Please migrate to ``'w'`` typecode. - -(2) .. versionadded:: 3.13 -(3) +(2) The IEEE 754 binary16 "half precision" type was introduced in the 2008 revision of the `IEEE 754 standard `_. This type is not widely supported by C compilers. It's available @@ -83,8 +66,8 @@ Notes: .. versionadded:: 3.15 -(4) - Complex types (``F``, ``D``, ``Zf`` and ``Zd``) are available unconditionally, +(3) + Complex types (``Zf`` and ``Zd``) are available unconditionally, regardless on support for complex types (the Annex G of the C11 standard) by the C compiler. As specified in the C11 standard, each complex type is represented by a @@ -95,7 +78,7 @@ Notes: .. seealso:: The :ref:`ctypes ` and - :ref:`struct ` modules, + :ref:`struct ` modules, as well as third-party modules like `numpy `__, use similar -- but slightly different -- type codes. @@ -111,7 +94,7 @@ The module defines the following item: A tuple with all available type codes. - .. versionchanged:: next + .. versionchanged:: 3.15 The type changed from :class:`str` to :class:`tuple`. @@ -138,6 +121,8 @@ The module defines the following type: :exc:`TypeError` is raised. Array objects also implement the buffer interface, and may be used wherever :term:`bytes-like objects ` are supported. + Arrays are :ref:`generic ` over the type of their contents. + .. audit-event:: array.__new__ typecode,initializer array.array @@ -224,7 +209,7 @@ The module defines the following type: .. method:: fromunicode(ustr, /) Extends this array with data from the given Unicode string. - The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. + The array must have type code ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an array of some other type. @@ -292,7 +277,7 @@ The module defines the following type: .. method:: tounicode() - Convert the array to a Unicode string. The array must have a type ``'u'`` or ``'w'``; + Convert the array to a Unicode string. The array must have a type ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to obtain a Unicode string from an array of some other type. @@ -300,7 +285,7 @@ The module defines the following type: The string representation of array objects has the form ``array(typecode, initializer)``. The *initializer* is omitted if the array is empty, otherwise it is -a Unicode string if the *typecode* is ``'u'`` or ``'w'``, otherwise it is +a Unicode string if the *typecode* is ``'w'``, otherwise it is a list of numbers. The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index e10b8e22df0fefb..4809fdb42bf3d77 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -168,7 +168,7 @@ Node classes arguments that were set as attributes of the AST node, even if they did not match any of the fields of the AST node. These cases now raise a :exc:`TypeError`. -.. deprecated-removed:: next 3.20 +.. deprecated-removed:: 3.15 3.20 In the :ref:`grammar above `, the AST node classes that correspond to production rules with variants (aka "sums") are abstract @@ -2548,7 +2548,7 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.15 Omit optional ``Load()`` values by default. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *color* parameter. @@ -2608,7 +2608,7 @@ Command-line usage .. versionadded:: 3.9 -.. versionchanged:: next +.. versionchanged:: 3.15 The output is now syntax highlighted by default. This can be :ref:`controlled using environment variables `. diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 43977de273e61f6..195d99123dbd367 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -101,6 +101,8 @@ Future Object implementations can inject their own optimized implementations of a Future object. + Futures are :ref:`generic ` over the type of their results. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 2e17d0dc70c7a5c..64f0810777e41b9 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -381,7 +381,7 @@ and reliable way to wait for all tasks in the group to finish. * pass the task group instance or bound :meth:`cancel` method to some other task *before* opening the task group, allowing remote cancellation - .. versionadded:: next + .. versionadded:: 3.15 Example:: @@ -1230,6 +1230,9 @@ Task object blocks. If the coroutine returns or raises without blocking, the task will be finished eagerly and will skip scheduling to the event loop. + Tasks are :ref:`generic ` over the return type of their wrapped + coroutines. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. diff --git a/Doc/library/asyncio-threading.rst b/Doc/library/asyncio-threading.rst new file mode 100644 index 000000000000000..526901a2e7eb206 --- /dev/null +++ b/Doc/library/asyncio-threading.rst @@ -0,0 +1,154 @@ +.. currentmodule:: asyncio + +.. _asyncio-threading: + +asyncio and free-threaded Python +================================ + +asyncio uses an event loop as a scheduler to enable highly efficient +concurrency by switching between tasks to allow non-blocking I/O +operations. This results in better performance for I/O-bound use +cases. It also allows off-loading CPU-bound work to a thread or +process pool, but that is still limited by the :term:`global +interpreter lock` in CPython. + +However, in :ref:`free-threaded Python `, +the GIL is disabled and Python can run true multi-threaded code. This +means that asyncio can now take advantage of multiple CPU cores without +the limitations imposed by the GIL. + +Since Python 3.14, asyncio has first-class support for free-threaded +Python, and the implementation of asyncio is safe to use in a +multi-threaded environment. + +A single event loop on one core can handle many connections +concurrently, but the Python code that runs to handle each one still +executes serially. Once requests involve a non-trivial amount of +per-request computation, that handling becomes the bottleneck, and a +single core can no longer keep up. Combining asyncio with threads is +most useful here: by running an event loop per thread, the handling of +different requests can run in parallel across multiple CPU cores. It is +also useful when you need to run blocking or CPU-bound code from an +asyncio application. + + +.. seealso:: + + `Scaling asyncio on Free-Threaded Python + `__, + a blog post by Kumar Aditya which explains the internal changes + that make asyncio safe and efficient under free-threaded Python, + together with benchmarks of the resulting improvements. + + +Thread safety considerations +---------------------------- + +While asyncio is designed to be thread-safe in a free-threaded Python +environment, there are still some considerations to keep in mind when +using asyncio with threads: + +1. **Event loop**: Each thread should have its own event loop which + should not be shared across threads. This ensures that the event loop + can manage its own tasks and callbacks without interference from + other threads. + +2. **Task management**: Tasks and futures created in one thread should + not be awaited or manipulated from another thread. + +3. **Thread-safe APIs**: When interacting with asyncio from multiple + threads, it's important to use thread-safe APIs provided by asyncio, + such as :func:`asyncio.run_coroutine_threadsafe` for submitting + coroutines to an event loop from another thread. If you need to + call a callback from a different thread, you can use + :meth:`loop.call_soon_threadsafe` to schedule it safely. + +4. **Synchronization**: The synchronization primitives provided by + asyncio (like :class:`asyncio.Lock` and :class:`asyncio.Event`) + are not designed to be used across threads. If you need to + synchronize between threads, you should use the synchronization + primitives from the :mod:`threading` module instead. + + +Using asyncio with threads +-------------------------- + +asyncio supports running one event loop per thread, which allows you to +take advantage of multiple CPU cores in a free-threaded Python +environment. Each thread can run its own event loop, and tasks can be +scheduled on those loops independently. + +Here's an example of how to use asyncio with threads:: + + import asyncio + import threading + + async def worker(name: str) -> None: + print(f"Worker {name} starting") + await asyncio.sleep(1) + print(f"Worker {name} done") + + def run_loop(name: str) -> None: + asyncio.run(worker(name)) + + threads = [ + threading.Thread(target=run_loop, args=(f"T{i}",)) + for i in range(4) + ] + for t in threads: + t.start() + for t in threads: + t.join() + +In this example, each thread creates its own event loop with +:func:`asyncio.run` and runs a coroutine on it. The threads execute +concurrently, and in a free-threaded build they can run on separate +CPU cores in parallel. + + +Producer/consumer across threads +-------------------------------- + +When a regular (non-asyncio) thread needs to hand work to an asyncio +event loop running in another thread, use a thread-safe primitive such +as :class:`queue.Queue` rather than :class:`asyncio.Queue`, which is +only safe within a single event loop.:: + + import asyncio + import queue + import threading + + def producer(q: queue.Queue[int]) -> None: + for i in range(5): + print(f"Producing {i}") + q.put(i) + q.shutdown() + + async def consumer(q: queue.Queue[int]) -> None: + while True: + try: + item = q.get_nowait() + except queue.Empty: + await asyncio.sleep(0.1) + continue + except queue.ShutDown: + break + print(f"Consumed {item}") + await asyncio.sleep(item) + + q: queue.Queue[int] = queue.Queue() + consumer_thread = threading.Thread( + target=lambda: asyncio.run(consumer(q)) + ) + consumer_thread.start() + producer(q) + consumer_thread.join() + +The producer runs on the main thread while the consumer runs inside an +event loop on its own thread, yet they communicate safely through +``queue.Queue``. When the queue is empty the consumer sleeps briefly +and tries again. When the producer is done it calls +:meth:`~queue.Queue.shutdown`, which causes subsequent +:meth:`~queue.Queue.get_nowait` calls to raise :exc:`queue.ShutDown` +so the consumer can exit cleanly. + diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 0f72e31dee5f1d1..90a465f3e1d3af4 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -128,6 +128,7 @@ for full functionality and the latest features. asyncio-api-index.rst asyncio-llapi-index.rst asyncio-dev.rst + asyncio-threading.rst .. note:: The source code for asyncio can be found in :source:`Lib/asyncio/`. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 32da8294c5a58a0..8af40a2f8a65e3f 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -16,8 +16,10 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. This includes the :ref:`encodings specified in ` -:rfc:`4648` (Base64, Base32 and Base16) -and the non-standard :ref:`Base85 encodings `. +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -284,19 +286,28 @@ POST request. Base85 Encodings ----------------- -Base85 encoding is not formally specified but rather a de facto standard, -thus different systems perform the encoding differently. +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. -The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of -the de facto standard. You should call the function with the Base85 -implementation used by the software you intend to work with. +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. -The two functions present in this module differ in how they handle the following: +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. -* Whether to include enclosing ``<~`` and ``~>`` markers -* Whether to include newline characters -* The set of ASCII characters used for encoding -* Handling of null bytes +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. Refer to the documentation of the individual functions for more information. @@ -307,18 +318,22 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 @@ -330,10 +345,12 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -345,7 +362,7 @@ Refer to the documentation of the individual functions for more information. .. versionadded:: 3.4 - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *canonical* parameter. Single-character final groups are now always rejected as encoding violations. @@ -356,8 +373,11 @@ Refer to the documentation of the individual functions for more information. Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -372,8 +392,7 @@ Refer to the documentation of the individual functions for more information. .. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -392,11 +411,12 @@ Refer to the documentation of the individual functions for more information. .. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -414,8 +434,7 @@ Refer to the documentation of the individual functions for more information. .. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -499,3 +518,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 8b4ba6ae9fb2549..ceb80a35a1a76bb 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -133,8 +133,11 @@ The :mod:`!binascii` module defines the following functions: should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is not supported by the "standard" Ascii85 encoding. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. If + *adobe* is true, a leading ``<~`` is optionally accepted, while a + trailing ``~>`` is *required*, and :exc:`binascii.Error` is raised + if it is not found. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. @@ -164,12 +167,16 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + If *pad* is true, the zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.15 @@ -213,8 +220,10 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + If *pad* is true, the zero-padding applied to the end of the input + is retained in the output, which will always be a multiple of 5 + bytes, and thus the length of the data may not be preserved on + decoding. .. versionadded:: 3.15 @@ -358,9 +367,8 @@ The :mod:`!binascii` module defines the following functions: *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. - Similar functionality (accepting only text string arguments, but more - liberal towards whitespace) is also accessible using the - :meth:`bytes.fromhex` class method. + Similar functionality (but more liberal towards whitespace) is also accessible + using the :meth:`bytes.fromhex` class method. .. versionchanged:: 3.15 Added the *ignorechars* parameter. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 2c29a5ec992737e..39ab75c0904df9c 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -129,7 +129,7 @@ thoughts in mind: they are used. Consequently, if the search functions are used in a loop, the key function may be called again and again on the same array elements. If the key function isn't fast, consider wrapping it with - :py:func:`functools.cache` to avoid duplicate computations. Alternatively, + :py:deco:`functools.cache` to avoid duplicate computations. Alternatively, consider searching an array of precomputed keys to locate the insertion point (as shown in the examples section below). @@ -200,7 +200,7 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score) + >>> def grade(score): ... i = bisect([60, 70, 80, 90], score) ... return "FDCBA"[i] ... diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 3eb3f741eee45a4..31faa8c4fb43dc6 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -756,7 +756,7 @@ The following options are accepted: By default, today's date is highlighted in color and can be :ref:`controlled using environment variables `. -.. versionchanged:: next +.. versionchanged:: 3.15 By default, the month is now also highlighted in color, and the days of the week are also in color. This behavior can be :ref:`controlled using environment variables `. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 9259ab10d5850b5..99fcf35aa893e42 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1155,7 +1155,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1192,7 +1192,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ @@ -1399,6 +1399,14 @@ encodings. | punycode | | Implement :rfc:`3492`. | | | | Stateful codecs are not | | | | supported. | +| | | | +| | | .. warning:: | +| | | | +| | | The decoding and | +| | | encoding algorithms | +| | | scale poorly, so | +| | | limit the length of | +| | | untrusted input. | +--------------------+---------+---------------------------+ | raw_unicode_escape | | Latin-1 encoding with | | | | :samp:`\\u{XXXX}` and | diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 51853725b1b297c..10e3790717ed6ed 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -456,7 +456,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define + inherit from both :class:`Set` and :class:`Hashable`, then define ``__hash__ = Set._hash``. .. seealso:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index e42bdc06be09fff..d09a6c92bbd37dc 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -484,6 +484,8 @@ or subtracting from an empty counter. Unix. They are also useful for tracking transactions and other pools of data where only the most recent activity is of interest. + Deques are :ref:`generic ` over the type of their contents. + Deque objects support the following methods: @@ -739,6 +741,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, as if they were passed to the :class:`dict` constructor, including keyword arguments. + :class:`!defaultdict`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the dictionary's keys and values. + :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: @@ -1223,12 +1228,12 @@ original insertion position is changed and moved to the end:: self.move_to_end(key) An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`: +variants of :deco:`functools.lru_cache`: .. testcode:: from collections import OrderedDict - from time import time + from time import monotonic class TimeBoundedLRU: "LRU Cache that invalidates and refreshes old entries." @@ -1243,10 +1248,10 @@ variants of :func:`functools.lru_cache`: if args in self.cache: self.cache.move_to_end(args) timestamp, result = self.cache[args] - if time() - timestamp <= self.maxage: + if monotonic() - timestamp <= self.maxage: return result result = self.func(*args) - self.cache[args] = time(), result + self.cache[args] = monotonic(), result if len(self.cache) > self.maxsize: self.cache.popitem(last=False) return result diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index d058ba206c6cd62..6f005f98bd3ede5 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,7 +22,7 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`~types.NoneType` type. + ``None`` is the sole instance of the :class:`~types.NoneType` type. .. data:: NotImplemented @@ -33,7 +33,7 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :class:`types.NotImplementedType` type. .. note:: @@ -68,7 +68,7 @@ A small number of constants live in the built-in namespace. They are: The same as the ellipsis literal "``...``", an object frequently used to indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but assignment to ``...`` raises a :exc:`SyntaxError`. - ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. + ``Ellipsis`` is the sole instance of the :class:`types.EllipsisType` type. .. data:: __debug__ diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 77bac8dcc3afbdb..666f85997448881 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -88,11 +88,11 @@ Functions and classes provided: the exception has been handled, and execution will resume with the statement immediately following the :keyword:`!with` statement. - :func:`contextmanager` uses :class:`ContextDecorator` so the context managers + :deco:`contextmanager` uses :class:`ContextDecorator` so the context managers it creates can be used as decorators as well as in :keyword:`with` statements. When used as a decorator, a new generator instance is implicitly created on each function call (this allows the otherwise "one-shot" context managers - created by :func:`contextmanager` to meet the requirement that context + created by :deco:`contextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators). .. versionchanged:: 3.2 @@ -101,7 +101,7 @@ Functions and classes provided: .. decorator:: asynccontextmanager - Similar to :func:`~contextlib.contextmanager`, but creates an + Similar to :deco:`~contextlib.contextmanager`, but creates an :ref:`asynchronous context manager `. This function is a :term:`decorator` that can be used to define a factory @@ -128,7 +128,7 @@ Functions and classes provided: .. versionadded:: 3.7 - Context managers defined with :func:`asynccontextmanager` can be used + Context managers defined with :deco:`asynccontextmanager` can be used either as decorators or with :keyword:`async with` statements:: import time @@ -148,11 +148,11 @@ Functions and classes provided: When used as a decorator, a new generator instance is implicitly created on each function call. This allows the otherwise "one-shot" context managers - created by :func:`asynccontextmanager` to meet the requirement that context + created by :deco:`asynccontextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators. .. versionchanged:: 3.10 - Async context managers created with :func:`asynccontextmanager` can + Async context managers created with :deco:`asynccontextmanager` can be used as decorators. @@ -400,7 +400,7 @@ Functions and classes provided: ``__exit__`` retains its optional exception handling even when used as a decorator. - ``ContextDecorator`` is used by :func:`contextmanager`, so you get this + ``ContextDecorator`` is used by :deco:`contextmanager`, so you get this functionality automatically. Example of ``ContextDecorator``:: @@ -482,7 +482,7 @@ Functions and classes provided: .. versionadded:: 3.2 - .. versionchanged:: next + .. versionchanged:: 3.15 Decorating a generator function, coroutine function, or asynchronous generator function now keeps the context manager open across iteration or await. Previously the context manager exited as soon as the @@ -538,7 +538,7 @@ Functions and classes provided: .. versionadded:: 3.10 - .. versionchanged:: next + .. versionchanged:: 3.15 Decorating an asynchronous generator function now keeps the context manager open across iteration. Previously the context manager exited as soon as the generator object was created. Synchronous functions @@ -702,7 +702,7 @@ Functions and classes provided: Similar to :meth:`ExitStack.close` but properly handles awaitables. - Continuing the example for :func:`asynccontextmanager`:: + Continuing the example for :deco:`asynccontextmanager`:: async with AsyncExitStack() as stack: connections = [await stack.enter_async_context(get_connection()) @@ -960,7 +960,7 @@ Files are an example of effectively single use context managers, since the first :keyword:`with` statement will close the file, preventing any further IO operations using that file object. -Context managers created using :func:`contextmanager` are also single use +Context managers created using :deco:`contextmanager` are also single use context managers, and will complain about the underlying generator failing to yield if an attempt is made to use them a second time:: diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 93d0c0d34bf039d..b0cc0be8e911bf0 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -42,6 +42,9 @@ Context Variables references to context variables which prevents context variables from being properly garbage collected. + :class:`!ContextVar`\s are :ref:`generic ` over the type of + their contained value. + .. attribute:: ContextVar.name The name of the variable. This is a read-only property. @@ -130,6 +133,9 @@ Context Variables Tokens support the :ref:`context manager protocol ` to automatically reset context variables. See :meth:`ContextVar.set`. + Tokens are :ref:`generic ` over the same type as the + :class:`ContextVar` which created them. + .. versionadded:: 3.14 Added support for usage as a context manager. diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 121c44a16ad43b9..39fc7800d03a916 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -72,9 +72,13 @@ file, socket, window, or any similar types. It does "copy" functions and classes (shallow and deeply), by returning the original object unchanged; this is compatible with the way these are treated by the :mod:`pickle` module. -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. +Shallow copies of many collections can be made using the corresponding +:meth:`!copy` method (such as :meth:`list.copy`, :meth:`dict.copy` or +:meth:`set.copy`), and of sequences (such as lists or bytearrays) by making +a slice of the entire sequence (``sequence[:]``). +However, these methods and slicing can create an instance of the base type +when copying an instance of a subclass, whereas :func:`copy.copy` normally +returns an instance of the same type. .. index:: pair: module; pickle diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 21ecdbcc08f3486..87c2d4702e74086 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -81,6 +81,13 @@ The :mod:`!csv` module defines the following functions: Spam, Spam, Spam, Spam, Spam, Baked Beans Spam, Lovely Spam, Wonderful Spam + where :file:`eggs.csv` contains: + + .. code-block:: text + + Spam Spam Spam Spam Spam |Baked Beans| + Spam |Lovely Spam| |Wonderful Spam| + .. function:: writer(csvfile, /, dialect='excel', **fmtparams) @@ -110,6 +117,13 @@ The :mod:`!csv` module defines the following functions: spamwriter.writerow(['Spam'] * 5 + ['Baked Beans']) spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) + which writes :file:`eggs.csv` containing: + + .. code-block:: text + + Spam Spam Spam Spam Spam |Baked Beans| + Spam |Lovely Spam| |Wonderful Spam| + .. function:: register_dialect(name, /, dialect='excel', **fmtparams) @@ -191,6 +205,14 @@ The :mod:`!csv` module defines the following classes: >>> print(row) {'first_name': 'John', 'last_name': 'Cleese'} + where :file:`names.csv` contains: + + .. code-block:: text + + first_name,last_name + Eric,Idle + John,Cleese + .. class:: DictWriter(f, fieldnames, restval='', extrasaction='raise', \ dialect='excel', *args, **kwds) @@ -228,6 +250,15 @@ The :mod:`!csv` module defines the following classes: writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'}) writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'}) + which writes :file:`names.csv` containing: + + .. code-block:: text + + first_name,last_name + Baked,Beans + Lovely,Spam + Wonderful,Spam + .. class:: Dialect diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 3db66ad8ea66e7d..51f08fdc0544c96 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,14 @@ used to wrap these libraries in pure Python. .. include:: ../includes/optional-module.rst +.. warning:: + + :mod:`!ctypes` provides low-level access to native libraries and the + process's memory, bypassing Python's safety mechanisms and allowing + execution of arbitrary native code. + Incorrect use can corrupt data and objects, reveal sensitive information, + cause crashes, or otherwise compromise the running process. + .. _ctypes-ctypes-tutorial: @@ -198,10 +206,8 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`!ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +The :mod:`faulthandler` module can help debug crashes, +such as segmentation faults produced by erroneous C library calls. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. @@ -380,7 +386,7 @@ in both C and ``libffi``, the following complex types are available: - :py:class:`complex` - ``'Zg'`` -.. versionchanged:: next +.. versionchanged:: 3.15 The :py:attr:`~_SimpleCData._type_` types ``F``, ``D`` and ``G`` have been replaced with ``Zf``, ``Zd`` and ``Zg``. @@ -545,7 +551,7 @@ object with an :attr:`!_as_parameter_` attribute:: >>> If you don't want to store the instance's data in the :attr:`!_as_parameter_` -instance variable, you could define a :class:`property` which makes the +instance variable, you could define a :deco:`property` which makes the attribute available on request. @@ -2572,8 +2578,7 @@ Fundamental data types .. attribute:: _type_ - Class attribute that contains an internal type code, as a - single-character string. + Class attribute that contains an internal type code, as a string. See :ref:`ctypes-fundamental-data-types` for a summary. Types marked \* in the summary may be (or always are) aliases of a @@ -2587,7 +2592,7 @@ Fundamental data types .. seealso:: - The :mod:`array` and :ref:`struct ` modules, + The :mod:`array` and :ref:`struct ` modules, as well as third-party modules like `numpy `__, use similar -- but slightly different -- type codes. @@ -3172,6 +3177,8 @@ Arrays and pointers subscript and slice accesses; for slice reads, the resulting object is *not* itself an :class:`Array`. + Arrays are :ref:`generic ` over the type of their elements. + .. attribute:: _length_ diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index 9ae82c144655383..3ac76edd38dcc8f 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -114,7 +114,7 @@ C library: .. function:: isblank(c) - Checks for an ASCII whitespace character; space or horizontal tab. + Checks for an ASCII blank character; space or horizontal tab. .. function:: iscntrl(c) @@ -130,7 +130,7 @@ C library: .. function:: isgraph(c) - Checks for ASCII any printable character except space. + Checks for any ASCII printable character except space. .. function:: islower(c) @@ -145,7 +145,7 @@ C library: .. function:: ispunct(c) - Checks for any printable ASCII character which is not a space or an alphanumeric + Checks for any ASCII printable character which is not a space or an alphanumeric character. @@ -168,7 +168,8 @@ C library: .. function:: isctrl(c) - Checks for an ASCII control character (ordinal values 0 to 31). + Checks for an ASCII control character (ordinal values 0 to 31). Unlike + :func:`iscntrl`, this does not include the delete character (0x7f). .. function:: ismeta(c) diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index 5bc6b74b7f07cae..dd345bff428ad68 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -16,6 +16,14 @@ displayed. Panels can be added, moved up or down in the stack, and removed. Functions --------- +The module :mod:`!curses.panel` defines the following exception: + + +.. exception:: error + + Exception raised when a curses panel library function returns an error. + + The module :mod:`!curses.panel` defines the following functions: @@ -26,7 +34,8 @@ The module :mod:`!curses.panel` defines the following functions: .. function:: new_panel(win) - Returns a panel object, associating it with the given window *win*. Be aware + Returns a panel object, associating it with the given window *win* and + placing the new panel on top of the panel stack. Be aware that you need to keep the returned panel object referenced explicitly. If you don't, the panel object is garbage collected and removed from the panel stack. @@ -44,75 +53,94 @@ The module :mod:`!curses.panel` defines the following functions: .. _curses-panel-objects: -Panel Objects +Panel objects ------------- -Panel objects, as returned by :func:`new_panel` above, are windows with a -stacking order. There's always a window associated with a panel which determines -the content, while the panel methods are responsible for the window's depth in -the panel stack. +.. raw:: html + + + + + + + + + + + + + + + +.. class:: panel + + Panel objects, as returned by :func:`new_panel` above, are windows with a + stacking order. There's always a window associated with a panel which + determines the content, while the panel methods are responsible for the + window's depth in the panel stack. -Panel objects have the following methods: + Panel objects have the following methods: -.. method:: Panel.above() +.. method:: panel.above() Returns the panel above the current panel. -.. method:: Panel.below() +.. method:: panel.below() Returns the panel below the current panel. -.. method:: Panel.bottom() +.. method:: panel.bottom() Push the panel to the bottom of the stack. -.. method:: Panel.hidden() +.. method:: panel.hidden() Returns ``True`` if the panel is hidden (not visible), ``False`` otherwise. -.. method:: Panel.hide() +.. method:: panel.hide() Hide the panel. This does not delete the object, it just makes the window on screen invisible. -.. method:: Panel.move(y, x) +.. method:: panel.move(y, x) Move the panel to the screen coordinates ``(y, x)``. -.. method:: Panel.replace(win) +.. method:: panel.replace(win) Change the window associated with the panel to the window *win*. -.. method:: Panel.set_userptr(obj) +.. method:: panel.set_userptr(obj) Set the panel's user pointer to *obj*. This is used to associate an arbitrary piece of data with the panel, and can be any Python object. -.. method:: Panel.show() +.. method:: panel.show() - Display the panel (which might have been hidden). + Display the panel (which might have been hidden), placing it on top of + the panel stack. -.. method:: Panel.top() +.. method:: panel.top() Push panel to the top of the stack. -.. method:: Panel.userptr() +.. method:: panel.userptr() Returns the user pointer for the panel. This might be any Python object. -.. method:: Panel.window() +.. method:: panel.window() Returns the window object associated with the panel. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 0f1449873fcf735..a8a9680d0703ecf 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -80,9 +80,25 @@ The module :mod:`!curses` defines the following functions: * Change the definition of the color-pair ``0`` to ``(fg, bg)``. + This is an ncurses extension. + .. versionadded:: 3.14 +.. function:: alloc_pair(fg, bg) + + Allocate a color pair for foreground color *fg* and background color *bg*, + and return its number. If a color pair for the same combination of colors + already exists, return its number. Otherwise allocate a new color pair and + return its number. + + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). + + .. versionadded:: next + + .. function:: baudrate() Return the output speed of the terminal in bits per second. On software @@ -116,7 +132,8 @@ The module :mod:`!curses` defines the following functions: Return the intensity of the red, green, and blue (RGB) components in the color *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, containing the R,G,B values for the given color, which will be between - ``0`` (no component) and ``1000`` (maximum amount of component). + ``0`` (no component) and ``1000`` (maximum amount of component). Raise an + exception if the color is not supported. .. function:: color_pair(pair_number) @@ -162,7 +179,7 @@ The module :mod:`!curses` defines the following functions: Update the physical screen. The curses library keeps two data structures, one representing the current physical screen contents and a virtual screen - representing the desired next state. The :func:`doupdate` ground updates the + representing the desired next state. The :func:`doupdate` function updates the physical screen to match the virtual screen. The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write @@ -191,16 +208,51 @@ The module :mod:`!curses` defines the following functions: the curses library itself. +.. function:: erasewchar() + + Return the user's current erase character as a one-character string. + This is the wide-character variant of :func:`erasechar`. Availability + depends on building Python against a wide-character-aware version of the + underlying curses library. + + .. versionadded:: next + + .. function:: filter() The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the + called. The effect is that, during the initialization, :envvar:`LINES` is set to ``1``; the capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` string is set to the value of ``cr``. The effect is that the cursor is confined to the current line, and so are screen updates. This may be used for enabling character-at-a-time line editing without touching the rest of the screen. +.. function:: nofilter() + + Undo the effect of a previous :func:`.filter` call. + Like :func:`.filter`, it must be called before :func:`initscr` (or + :func:`newterm`) so that the next initialization uses the full screen + again. + + Availability: if the underlying curses library provides ``nofilter()``. + + .. versionadded:: next + + +.. function:: find_pair(fg, bg) + + Return the number of a color pair for foreground color *fg* and background + color *bg*, or ``-1`` if no color pair for this combination of colors has + been allocated. + + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). + + .. versionadded:: next + + .. function:: flash() Flash the screen. That is, change it to reverse-video and then change it back @@ -214,6 +266,18 @@ The module :mod:`!curses` defines the following functions: by the user and has not yet been processed by the program. +.. function:: free_pair(pair_number) + + Free the color pair *pair_number*, which must have been allocated by + :func:`alloc_pair`. The pair must not be in use. + + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). + + .. versionadded:: next + + .. function:: getmouse() After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this @@ -240,9 +304,10 @@ The module :mod:`!curses` defines the following functions: .. function:: getwin(file) - Read window related data stored in the file by an earlier :func:`window.putwin` call. + Read window-related data stored in the file by an earlier :meth:`window.putwin` call. The routine then creates and initializes a new window using that data, returning - the new window object. + the new window object. The *file* argument must be a file object opened for + reading in binary mode. .. function:: has_colors() @@ -253,7 +318,7 @@ The module :mod:`!curses` defines the following functions: Return ``True`` if the module supports extended colors; otherwise, return ``False``. Extended color support allows more than 256 color pairs for - terminals that support more than 16 colors (e.g. xterm-256color). + terminals that support more than 16 colors (for example, xterm-256color). Extended color support requires ncurses version 6.1 or later. @@ -326,6 +391,48 @@ The module :mod:`!curses` defines the following functions: cause the interpreter to exit. +.. function:: intrflush(flag) + + If *flag* is ``True``, pressing an interrupt key (interrupt, break, or quit) + will flush all output in the terminal driver queue. If *flag* is ``False``, + no flushing is done. + + +.. function:: is_cbreak() + + Return ``True`` if cbreak mode (see :func:`cbreak`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. + + .. versionadded:: next + + +.. function:: is_echo() + + Return ``True`` if echo mode (see :func:`echo`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. + + .. versionadded:: next + + +.. function:: is_nl() + + Return ``True`` if nl mode (see :func:`nl`) is enabled, ``False`` otherwise. + Availability: ncurses 6.5 or later. + + .. versionadded:: next + + +.. function:: is_raw() + + Return ``True`` if raw mode (see :func:`raw`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. + + .. versionadded:: next + + .. function:: is_term_resized(nlines, ncols) Return ``True`` if :func:`resize_term` would modify the window structure, @@ -347,6 +454,8 @@ The module :mod:`!curses` defines the following functions: bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding ASCII character. + Raise a :exc:`ValueError` if *k* is negative. + .. function:: killchar() @@ -355,6 +464,16 @@ The module :mod:`!curses` defines the following functions: by the curses library itself. +.. function:: killwchar() + + Return the user's current line kill character as a one-character string. + This is the wide-character variant of :func:`killchar`. Availability + depends on building Python against a wide-character-aware version of the + underlying curses library. + + .. versionadded:: next + + .. function:: longname() Return a bytes object containing the terminfo long name field describing the current @@ -372,7 +491,8 @@ The module :mod:`!curses` defines the following functions: Set the maximum time in milliseconds that can elapse between press and release events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 milliseconds, or one fifth of a second. + interval value. The default value is 166 milliseconds, or one sixth of a second. + Use a negative *interval* to obtain the interval value without changing it. .. function:: mousemask(mousemask) @@ -380,7 +500,7 @@ The module :mod:`!curses` defines the following functions: Set the mouse events to be reported, and return a tuple ``(availmask, oldmask)``. *availmask* indicates which of the specified mouse events can be reported; on complete failure it returns ``0``. *oldmask* is the previous value of - the given window's mouse event mask. If this function is never called, no mouse + the mouse event mask. If this function is never called, no mouse events are ever reported. @@ -402,7 +522,7 @@ The module :mod:`!curses` defines the following functions: methods of a pad require 6 arguments to specify the part of the pad to be displayed and the location on the screen to be used for the display. The arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* - arguments refer to the upper left corner of the pad region to be displayed and + arguments refer to the upper-left corner of the pad region to be displayed and the *s* arguments define a clipping box on the screen within which the pad region is to be displayed. @@ -417,12 +537,44 @@ The module :mod:`!curses` defines the following functions: right corner of the screen. -.. function:: nl() +.. function:: newterm(type=None, fd=None, infd=None, /) + + Initialize a new terminal in addition to the one initialized by + :func:`initscr`, + and return a :ref:`screen ` for it. + This allows a program to drive more than one terminal. + + *type* is the terminal name, as in :func:`setupterm`; + if ``None``, the value of the :envvar:`TERM` environment variable is used. + *fd* and *infd* are the output and input files for the terminal: + either a file object or a file descriptor. + They default to :data:`sys.stdout` and :data:`sys.stdin`. + + The new screen becomes the current one. + Use :func:`set_term` to switch between screens. + + .. versionadded:: next + + +.. function:: new_prescr() + + Return a new :ref:`screen ` + that can be used to call functions that affect global state + before :func:`initscr` or :func:`newterm` is called. + + Availability: if the underlying curses library provides ``new_prescr()``. + + .. versionadded:: next + + +.. function:: nl(flag=True) Enter newline mode. This mode translates the return key into newline on input, and translates newline into return and line-feed on output. Newline mode is initially on. + If *flag* is ``False``, the effect is the same as calling :func:`nonl`. + .. function:: nocbreak() @@ -475,6 +627,8 @@ The module :mod:`!curses` defines the following functions: terminfo capability for the current terminal. Note that the output of :func:`putp` always goes to standard output. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: qiflush([flag]) @@ -490,6 +644,18 @@ The module :mod:`!curses` defines the following functions: presented to curses input functions one by one. +.. function:: reset_color_pairs() + + Discard all color-pair definitions, releasing the color pairs allocated by + :func:`init_pair` and :func:`alloc_pair`. + + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). + + .. versionadded:: next + + .. function:: reset_prog_mode() Restore the terminal to "program" mode, as previously saved by @@ -557,6 +723,17 @@ The module :mod:`!curses` defines the following functions: .. versionadded:: 3.9 + +.. function:: set_term(screen, /) + + Make *screen*, a :ref:`screen ` returned by + :func:`newterm`, the current terminal, + and return the previously current screen. + Returns ``None`` if the previous screen was the one created by + :func:`initscr`. + + .. versionadded:: next + .. function:: setsyx(y, x) Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then @@ -571,6 +748,10 @@ The module :mod:`!curses` defines the following functions: file descriptor to which any initialization sequences will be sent; if not supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. + Raise a :exc:`curses.error` if the terminal could not be found or its + terminfo database entry could not be read. If the terminal has already + been initialized, this function has no effect. + .. function:: start_color() @@ -605,6 +786,8 @@ The module :mod:`!curses` defines the following functions: Boolean capability, or ``0`` if it is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tigetnum(capname) @@ -613,6 +796,8 @@ The module :mod:`!curses` defines the following functions: numeric capability, or ``-1`` if it is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tigetstr(capname) @@ -621,13 +806,17 @@ The module :mod:`!curses` defines the following functions: is not a terminfo "string capability", or is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tparm(str[, ...]) Instantiate the bytes object *str* with the supplied parameters, where *str* should - be a parameterized string obtained from the terminfo database. E.g. + be a parameterized string obtained from the terminfo database. For example, ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact - result depending on terminal type. + result depending on terminal type. Up to nine integer parameters may be supplied. + + :func:`setupterm` (or :func:`initscr`) must be called first. .. function:: typeahead(fd) @@ -649,6 +838,18 @@ The module :mod:`!curses` defines the following functions: example as ``b'^C'``. Printing characters are left as they are. +.. function:: wunctrl(ch) + + Return a string which is a printable representation of the wide character *ch*. + Control characters are represented as a caret followed by the character, for + example as ``'^C'``. Printing characters are left as they are. This is the + wide-character variant of :func:`unctrl`, returning a :class:`str` rather than + :class:`bytes`. Availability depends on building Python against a + wide-character-aware version of the underlying curses library. + + .. versionadded:: next + + .. function:: ungetch(ch) Push *ch* so the next :meth:`~window.getch` will return it. @@ -713,7 +914,7 @@ The module :mod:`!curses` defines the following functions: .. _curses-window-objects: -Window Objects +Window objects -------------- .. class:: window @@ -729,12 +930,19 @@ Window Objects character previously painted at that location. By default, the character position and attributes are the current settings for the window object. + *ch* may be a single character, optionally followed by combining + characters, that together occupy one character cell. + .. note:: Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, + Attempting to write to the lower-right corner of a window, subwindow, or pad will cause an exception to be raised after the character is printed. + .. versionchanged:: next + A character may now be given as a string of a base character followed + by combining characters, instead of only a single character. + .. method:: window.addnstr(str, n[, attr]) window.addnstr(y, x, str, n[, attr]) @@ -753,15 +961,14 @@ Window Objects .. note:: * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, + Attempting to write to the lower-right corner of a window, subwindow, or pad will cause an exception to be raised after the string is printed. - * A `bug in ncurses `_, the backend - for this Python module, can cause SegFaults when resizing windows. This - is fixed in ncurses-6.1-20190511. If you are stuck with an earlier - ncurses, you can avoid triggering this if you do not call :func:`addstr` - with a *str* that has embedded newlines. Instead, call :func:`addstr` - separately for each line. + * A bug in ncurses, the backend for this Python module, could cause + segfaults when resizing windows. This was fixed in ncurses-6.1-20190511. + If you are stuck with an earlier ncurses, you can avoid triggering it by + not calling :meth:`!addstr` with a *str* that has embedded newlines; + instead, call :meth:`!addstr` separately for each line. .. method:: window.attroff(attr) @@ -794,6 +1001,9 @@ Window Objects * Wherever the former background character appears, it is changed to the new background character. + .. versionchanged:: next + Wide and combining characters are now accepted. + .. method:: window.bkgdset(ch[, attr]) @@ -804,6 +1014,9 @@ Window Objects characters. The background becomes a property of the character and moves with the character through any scrolling and insert/delete line/character operations. + .. versionchanged:: next + Wide and combining characters are now accepted. + .. method:: window.border([ls[, rs[, ts[, bs[, tl[, tr[, bl[, br]]]]]]]]) @@ -837,12 +1050,20 @@ Window Objects | *br* | Bottom-right corner | :const:`ACS_LRCORNER` | +-----------+---------------------+-----------------------+ + .. versionchanged:: next + Wide and combining characters are now accepted. A single call cannot mix + them with integer or byte characters. + .. method:: window.box([vertch, horch]) Similar to :meth:`border`, but both *ls* and *rs* are *vertch* and both *ts* and *bs* are *horch*. The default corner characters are always used by this function. + .. versionchanged:: next + Wide and combining characters are now accepted. A single call cannot mix + them with integer or byte characters. + .. method:: window.chgat(attr) window.chgat(num, attr) @@ -888,7 +1109,8 @@ Window Objects .. method:: window.delch([y, x]) - Delete any character at ``(y, x)``. + Delete the character under the cursor, or at ``(y, x)`` if specified. All + characters to the right on the same line are shifted one position left. .. method:: window.deleteln() @@ -910,6 +1132,9 @@ Window Objects Add character *ch* with attribute *attr*, and immediately call :meth:`refresh` on the window. + .. versionchanged:: next + Wide and combining characters are now accepted. + .. method:: window.enclose(y, x) @@ -964,6 +1189,16 @@ Window Objects .. versionadded:: 3.3 +.. method:: window.getdelay() + + Return the window's read timeout in milliseconds, + as set by :meth:`nodelay` or :meth:`timeout`: + ``-1`` for blocking, ``0`` for non-blocking, + or a positive number of milliseconds. + + .. versionadded:: next + + .. method:: window.getkey([y, x]) Get a character, returning a string instead of an integer, as :meth:`getch` @@ -977,6 +1212,14 @@ Window Objects Return a tuple ``(y, x)`` of the height and width of the window. +.. method:: window.getparent() + + Return the parent window of this subwindow, + or ``None`` if this window is not a subwindow. + + .. versionadded:: next + + .. method:: window.getparyx() Return the beginning coordinates of this window relative to its parent window @@ -984,29 +1227,56 @@ Window Objects parent. +.. method:: window.getscrreg() + + Return a tuple ``(top, bottom)`` of the window's current scrolling region, + as set by :meth:`setscrreg`. + + .. versionadded:: next + + .. method:: window.getstr() window.getstr(n) window.getstr(y, x) window.getstr(y, x, n) Read a bytes object from the user, with primitive line editing capacity. - The maximum value for *n* is 2047. + At most *n* characters are read; + *n* defaults to and cannot exceed 2047. .. versionchanged:: 3.14 The maximum value for *n* was increased from 1023 to 2047. +.. method:: window.get_wstr() + window.get_wstr(n) + window.get_wstr(y, x) + window.get_wstr(y, x, n) + + Read a string from the user, with primitive line editing capacity. + This is the wide-character variant of :meth:`getstr`: it returns a + :class:`str` rather than a :class:`bytes` object, so it can return + characters that are not representable in the window's encoding. + At most *n* characters are read; *n* defaults to and cannot exceed 2047. + + .. versionadded:: next + + .. method:: window.getyx() Return a tuple ``(y, x)`` of current cursor position relative to the window's upper-left corner. -.. method:: window.hline(ch, n) - window.hline(y, x, ch, n) +.. method:: window.hline(ch, n[, attr]) + window.hline(y, x, ch, n[, attr]) Display a horizontal line starting at ``(y, x)`` with length *n* consisting of - the character *ch*. + the character *ch* with attributes *attr*. The line stops at the right edge + of the window if fewer than *n* cells are available. + + .. versionchanged:: next + Wide and combining characters are now accepted. .. method:: window.idcok(flag) @@ -1019,8 +1289,8 @@ Window Objects .. method:: window.idlok(flag) - If *flag* is ``True``, :mod:`!curses` will try and use hardware line - editing facilities. Otherwise, line insertion/deletion are disabled. + If *flag* is ``True``, :mod:`!curses` will try to use hardware line + editing facilities. Otherwise, curses will not use them. .. method:: window.immedok(flag) @@ -1040,8 +1310,13 @@ Window Objects .. method:: window.insch(ch[, attr]) window.insch(y, x, ch[, attr]) - Paint character *ch* at ``(y, x)`` with attributes *attr*, moving the line from - position *x* right by one character. + Insert character *ch* with attributes *attr* before the character under the + cursor, or at ``(y, x)`` if specified. All characters to the right of the + cursor are shifted one position right, with the rightmost character on the + line being lost. The cursor position does not change. + + .. versionchanged:: next + Wide and combining characters are now accepted. .. method:: window.insdelln(nlines) @@ -1082,7 +1357,8 @@ Window Objects window.instr(y, x[, n]) Return a bytes object of characters, extracted from the window starting at the - current cursor position, or at *y*, *x* if specified. Attributes are stripped + current cursor position, or at *y*, *x* if specified, and stopping at the end + of the line. Attributes and color information are stripped from the characters. If *n* is specified, :meth:`instr` returns a string at most *n* characters long (exclusive of the trailing NUL). The maximum value for *n* is 2047. @@ -1091,6 +1367,61 @@ Window Objects The maximum value for *n* was increased from 1023 to 2047. +.. method:: window.in_wstr([n]) + window.in_wstr(y, x[, n]) + + Return a string of characters, extracted from the window starting at the + current cursor position, or at *y*, *x* if specified. This is the + wide-character variant of :meth:`instr`: it returns a :class:`str` rather + than a :class:`bytes` object, so it can return characters that are not + representable in the window's encoding. Attributes and color information + are stripped from the characters. The maximum value for *n* is 2047. + + .. versionadded:: next + + +.. method:: window.is_cleared() + + Return the current value set by :meth:`clearok`. + + .. versionadded:: next + + +.. method:: window.is_idcok() + + Return the current value set by :meth:`idcok`. + + .. versionadded:: next + + +.. method:: window.is_idlok() + + Return the current value set by :meth:`idlok`. + + .. versionadded:: next + + +.. method:: window.is_immedok() + + Return the current value set by :meth:`immedok`. + + .. versionadded:: next + + +.. method:: window.is_keypad() + + Return the current value set by :meth:`keypad`. + + .. versionadded:: next + + +.. method:: window.is_leaveok() + + Return the current value set by :meth:`leaveok`. + + .. versionadded:: next + + .. method:: window.is_linetouched(line) Return ``True`` if the specified line was modified since the last call to @@ -1098,6 +1429,49 @@ Window Objects exception if *line* is not valid for the given window. +.. method:: window.is_nodelay() + + Return the current value set by :meth:`nodelay`. + + .. versionadded:: next + + +.. method:: window.is_notimeout() + + Return the current value set by :meth:`notimeout`. + + .. versionadded:: next + + +.. method:: window.is_pad() + + Return ``True`` if the window is a pad created by :func:`newpad`. + + .. versionadded:: next + + +.. method:: window.is_scrollok() + + Return the current value set by :meth:`scrollok`. + + .. versionadded:: next + + +.. method:: window.is_subwin() + + Return ``True`` if the window is a subwindow created by :meth:`subwin` + or :meth:`derwin`. + + .. versionadded:: next + + +.. method:: window.is_syncok() + + Return the current value set by :meth:`syncok`. + + .. versionadded:: next + + .. method:: window.is_wintouched() Return ``True`` if the specified window was modified since the last call to @@ -1114,8 +1488,7 @@ Window Objects .. method:: window.leaveok(flag) If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor - position." This reduces cursor movement where possible. If possible the cursor - will be made invisible. + position." This reduces cursor movement where possible. If *flag* is ``False``, cursor will always be at "cursor position" after an update. @@ -1136,6 +1509,9 @@ Window Objects Move the window so its upper-left corner is at ``(new_y, new_x)``. + Moving the window so that any part of it would be off the screen is an error: + the window is not moved and :exc:`curses.error` is raised. + .. method:: window.nodelay(flag) @@ -1151,11 +1527,16 @@ Window Objects .. method:: window.noutrefresh() + window.noutrefresh(pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol) Mark for refresh but wait. This function updates the data structure representing the desired state of the window, but does not force an update of the physical screen. To accomplish that, call :func:`doupdate`. + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`; they have the same meaning as for + :meth:`refresh`. + .. method:: window.overlay(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) @@ -1206,12 +1587,12 @@ Window Objects Update the display immediately (sync actual screen with previous drawing/deleting methods). - The 6 optional arguments can only be specified when the window is a pad created - with :func:`newpad`. The additional parameters are needed to indicate what part - of the pad and screen are involved. *pminrow* and *pmincol* specify the upper - left-hand corner of the rectangle to be displayed in the pad. *sminrow*, + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`. The additional parameters are needed to indicate what part + of the pad and screen are involved. *pminrow* and *pmincol* specify the + upper-left corner of the rectangle to be displayed in the pad. *sminrow*, *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be - displayed on the screen. The lower right-hand corner of the rectangle to be + displayed on the screen. The lower-right corner of the rectangle to be displayed in the pad is calculated from the screen coordinates, since the rectangles must be the same size. Both rectangles must be entirely contained within their respective structures. Negative values of *pminrow*, *pmincol*, @@ -1228,7 +1609,9 @@ Window Objects .. method:: window.scroll([lines=1]) - Scroll the screen or scrolling region upward by *lines* lines. + Scroll the screen or scrolling region. Scroll upward by *lines* lines if + *lines* is positive, or downward if it is negative. Scrolling has no effect + unless it has been enabled for the window with :meth:`scrollok`. .. method:: window.scrollok(flag) @@ -1261,15 +1644,17 @@ Window Objects .. method:: window.subpad(begin_y, begin_x) window.subpad(nlines, ncols, begin_y, begin_x) - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Return a sub-pad, whose upper-left corner is at ``(begin_y, begin_x)``, and + whose width/height is *ncols*/*nlines*. The coordinates are relative to the + parent pad (unlike :meth:`subwin`, which uses screen coordinates). This + method is only available for pads created with :func:`newpad`. .. method:: window.subwin(begin_y, begin_x) window.subwin(nlines, ncols, begin_y, begin_x) - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Return a sub-window, whose upper-left corner is at the screen-relative + coordinates ``(begin_y, begin_x)``, and whose width/height is *ncols*/*nlines*. By default, the sub-window will extend from the specified position to the lower right corner of the window. @@ -1323,12 +1708,81 @@ Window Objects :meth:`refresh`. +.. method:: window.use(func, /, *args, **kwargs) + + Call ``func(window, *args, **kwargs)`` with the lock of the window held, + and return its result. + This provides automatic protection for the window + against concurrent access from another thread. + + Availability: if the underlying curses library provides ``use_window()``. + + .. versionadded:: next + + .. method:: window.vline(ch, n[, attr]) window.vline(y, x, ch, n[, attr]) Display a vertical line starting at ``(y, x)`` with length *n* consisting of the character *ch* with attributes *attr*. + .. versionchanged:: next + Wide and combining characters are now accepted. + + +.. _curses-screen-objects: + +Screen objects +-------------- + +.. class:: screen + + A *screen* object represents a terminal initialized by :func:`newterm` + (or :func:`new_prescr`), + in addition to the default screen created by :func:`initscr`. + Screen objects are returned by those functions; + they cannot be instantiated directly. + + A screen is freed automatically once it is no longer referenced, + either directly or through one of its windows. + Each window keeps its screen alive, + so a screen remains valid as long as any of its windows does. + + .. versionadded:: next + + +.. method:: screen.close() + + Detach the screen's standard window, + breaking the reference cycle between them + so the screen can be reclaimed promptly instead of waiting for a + garbage collection. + Afterwards :attr:`~screen.stdscr` is ``None`` + and the window it returned earlier can no longer be used. + The screen's resources are released + once it and all its windows are no longer referenced. + + .. versionadded:: next + + +.. attribute:: screen.stdscr + + The standard :ref:`window ` of the screen, + covering the whole terminal, + or ``None`` for a screen created by :func:`new_prescr`. + + +.. method:: screen.use(func, /, *args, **kwargs) + + Call ``func(screen, *args, **kwargs)`` with the lock of the screen held, + and return its result. + This provides automatic protection for the screen + against concurrent access from another thread. + + Availability: if the underlying curses library provides ``use_screen()``. + + .. versionadded:: next + Constants --------- @@ -1376,14 +1830,14 @@ The :mod:`!curses` module defines the following data members: .. data:: COLS - The width of the screen, i.e., the number of columns. + The width of the screen, that is, the number of columns. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. .. data:: LINES - The height of the screen, i.e., the number of lines. + The height of the screen, that is, the number of lines. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. @@ -1686,7 +2140,7 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | ACS code | Meaning | +========================+==========================================+ -| .. data:: ACS_BBSS | alternate name for upper right corner | +| .. data:: ACS_BBSS | alternate name for upper-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_BLOCK | solid square block | +------------------------+------------------------------------------+ @@ -1694,7 +2148,7 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_BSBS | alternate name for horizontal line | +------------------------+------------------------------------------+ -| .. data:: ACS_BSSB | alternate name for upper left corner | +| .. data:: ACS_BSSB | alternate name for upper-left corner | +------------------------+------------------------------------------+ | .. data:: ACS_BSSS | alternate name for top tee | +------------------------+------------------------------------------+ @@ -1720,9 +2174,9 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_LEQUAL | less-than-or-equal-to | +------------------------+------------------------------------------+ -| .. data:: ACS_LLCORNER | lower left-hand corner | +| .. data:: ACS_LLCORNER | lower-left corner | +------------------------+------------------------------------------+ -| .. data:: ACS_LRCORNER | lower right-hand corner | +| .. data:: ACS_LRCORNER | lower-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_LTEE | left tee | +------------------------+------------------------------------------+ @@ -1746,13 +2200,13 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_S9 | scan line 9 | +------------------------+------------------------------------------+ -| .. data:: ACS_SBBS | alternate name for lower right corner | +| .. data:: ACS_SBBS | alternate name for lower-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_SBSB | alternate name for vertical line | +------------------------+------------------------------------------+ | .. data:: ACS_SBSS | alternate name for right tee | +------------------------+------------------------------------------+ -| .. data:: ACS_SSBB | alternate name for lower left corner | +| .. data:: ACS_SSBB | alternate name for lower-left corner | +------------------------+------------------------------------------+ | .. data:: ACS_SSBS | alternate name for bottom tee | +------------------------+------------------------------------------+ @@ -1766,9 +2220,9 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_UARROW | up arrow | +------------------------+------------------------------------------+ -| .. data:: ACS_ULCORNER | upper left corner | +| .. data:: ACS_ULCORNER | upper-left corner | +------------------------+------------------------------------------+ -| .. data:: ACS_URCORNER | upper right corner | +| .. data:: ACS_URCORNER | upper-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_VLINE | vertical line | +------------------------+------------------------------------------+ @@ -1792,7 +2246,7 @@ The following table lists mouse button constants used by :meth:`getmouse`: +----------------------------------+---------------------------------------------+ | .. data:: BUTTON_CTRL | Control was down during button state change | +----------------------------------+---------------------------------------------+ -| .. data:: BUTTON_ALT | Control was down during button state change | +| .. data:: BUTTON_ALT | Alt was down during button state change | +----------------------------------+---------------------------------------------+ .. versionchanged:: 3.10 @@ -1841,9 +2295,9 @@ The module :mod:`!curses.textpad` defines the following function: Draw a rectangle. The first argument must be a window object; the remaining arguments are coordinates relative to that window. The second and third - arguments are the y and x coordinates of the upper left hand corner of the + arguments are the y and x coordinates of the upper-left corner of the rectangle to be drawn; the fourth and fifth arguments are the y and x - coordinates of the lower right hand corner. The rectangle will be drawn using + coordinates of the lower-right corner. The rectangle will be drawn using VT100/IBM PC forms characters on terminals that make this possible (including xterm and most other software terminal emulators). Otherwise it will be drawn with ASCII dashes, vertical bars, and plus signs. @@ -1857,32 +2311,36 @@ Textbox objects You can instantiate a :class:`Textbox` object as follows: -.. class:: Textbox(win) +.. class:: Textbox(win, insert_mode=False) Return a textbox widget object. The *win* argument should be a curses :ref:`window ` object in which the textbox is to - be contained. The edit cursor of the textbox is initially located at the - upper left hand corner of the containing window, with coordinates ``(0, 0)``. + be contained. If *insert_mode* is true, the textbox inserts typed + characters, shifting existing text to the right, rather than overwriting it. + The edit cursor of the textbox is initially located at the + upper-left corner of the containing window, with coordinates ``(0, 0)``. The instance's :attr:`stripspaces` flag is initially on. :class:`Textbox` objects have the following methods: - .. method:: edit([validator]) + .. method:: edit(validate=None) This is the entry point you will normally use. It accepts editing keystrokes until one of the termination keystrokes is entered. If - *validator* is supplied, it must be a function. It will be called for + *validate* is supplied, it must be a function. It will be called for each keystroke entered with the keystroke as a parameter; command dispatch - is done on the result. This method returns the window contents as a + is done on the result. If it returns a false value, the keystroke is + ignored. This method returns the window contents as a string; whether blanks in the window are included is affected by the :attr:`stripspaces` attribute. .. method:: do_command(ch) - Process a single command keystroke. Here are the supported special - keystrokes: + Process a single command keystroke. Returns ``1`` to continue editing, + or ``0`` if a termination keystroke was processed. Here are the supported + special keystrokes: +------------------+-------------------------------------------+ | Keystroke | Action | @@ -1905,7 +2363,8 @@ You can instantiate a :class:`Textbox` object as follows: | :kbd:`Control-H` | Delete character backward. | +------------------+-------------------------------------------+ | :kbd:`Control-J` | Terminate if the window is 1 line, | - | | otherwise insert newline. | + | | otherwise move to the start of the next | + | | line. | +------------------+-------------------------------------------+ | :kbd:`Control-K` | If line is blank, delete it, otherwise | | | clear to end of line. | diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 0bce3e5b762b8be..ce59d89843e761d 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -100,12 +100,25 @@ Module contents ignored. - *eq*: If true (the default), an :meth:`~object.__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. + generated. - If the class already defines :meth:`!__eq__`, this parameter is - ignored. + This method compares the class by comparing each field in order. Both + instances in the comparison must be of the identical type. + + If the class already defines :meth:`!__eq__`, this parameter is ignored. + + .. versionchanged:: 3.13 + The generated ``__eq__`` method now compares each field individually + (for example, ``self.a == other.a and self.b == other.b``), rather than + comparing tuples of fields as in previous versions. + + This change makes the comparison faster but it may alter results in cases + where attributes compare equal by identity but not by value (such as + ``float('nan')``). + + In Python 3.12 and earlier, the comparison was performed by creating + tuples of the fields and comparing them (for example, + ``(self.a, self.b) == (other.a, other.b)``). - *order*: If true (the default is ``False``), :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be @@ -418,7 +431,7 @@ Module contents :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, qualname=None, decorator=dataclass) Creates a new dataclass with name *cls_name*, fields as defined in *fields*, base classes as given in *bases*, and initialized @@ -434,6 +447,9 @@ Module contents of the dataclass is set to that value. By default, it is set to the module name of the caller. + If *qualname* is defined, the :attr:`~type.__qualname__` attribute of the dataclass + is set to that value. By default, it is set to the value passed to *cls_name*. + The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments as :deco:`dataclass`. By default, the :deco:`dataclass` @@ -464,6 +480,8 @@ Module contents .. versionadded:: 3.14 Added the *decorator* parameter. + .. versionadded:: next + Added the *qualname* parameter. .. function:: replace(obj, /, **changes) @@ -498,7 +516,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index 5d522556235a02b..74b23eaf87b4a0b 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -1,4 +1,4 @@ -Tkinter Dialogs +Tkinter dialogs =============== :mod:`!tkinter.simpledialog` --- Standard Tkinter input dialogs @@ -15,16 +15,41 @@ The :mod:`!tkinter.simpledialog` module contains convenience classes and functions for creating simple modal dialogs to get a value from the user. -.. function:: askfloat(title, prompt, **kw) - askinteger(title, prompt, **kw) - askstring(title, prompt, **kw) +.. function:: askfloat(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None, use_ttk=True) + askinteger(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None, use_ttk=True) + askstring(title, prompt, *, initialvalue=None, show=None, parent=None, use_ttk=True) - The above three functions provide dialogs that prompt the user to enter a value - of the desired type. + Prompt the user to enter a value of the desired type and return it, or + ``None`` if the dialog is cancelled. -.. class:: Dialog(parent, title=None) + *title* is the dialog title and *prompt* the message shown above the entry. + *initialvalue* is the value initially placed in the entry. + *parent* is the window over which the dialog is shown. + :func:`askinteger` and :func:`askfloat` also accept *minvalue* and + *maxvalue*, which bound the accepted value. + :func:`askstring` also accepts *show*, a character used to mask the entered + text, for example ``'*'`` to hide a password. + They use the themed :mod:`tkinter.ttk` widgets; pass ``use_ttk=False`` for + the classic widgets. + +.. class:: Dialog(parent, title=None, *, use_ttk=False) The base class for custom dialogs. + Instantiating it shows the dialog modally and returns once the user closes + it; the entered value is then available in the :attr:`!result` attribute. + When *use_ttk* is false (the default), the dialog is built from the classic + :mod:`tkinter` widgets, modelled on the classic ``tk_dialog``; when true, + from the themed :mod:`tkinter.ttk` widgets, modelled on the Tk message box. + The default is classic for compatibility, since the themed widgets set a + themed background that classic widgets added in :meth:`body` would not match. + + .. versionchanged:: next + Added the *use_ttk* parameter. + + .. attribute:: result + + The value produced by :meth:`apply`, or ``None`` if the dialog was + cancelled. .. method:: body(master) @@ -36,6 +61,61 @@ functions for creating simple modal dialogs to get a value from the user. Default behaviour adds OK and Cancel buttons. Override for custom button layouts. + .. method:: validate() + + Validate the data entered by the user. + Return true if it is valid, in which case the dialog proceeds to + :meth:`apply`; return false to keep the dialog open. + The default implementation always returns true; override it to check the + input. + + .. method:: apply() + + Process the data entered by the user, for example by storing it in the + :attr:`!result` attribute. + Called after :meth:`validate` succeeds and just before the dialog is + destroyed. + The default implementation does nothing; override it to act on or store + the result. + + .. method:: destroy() + + Destroy the dialog window, clearing the reference to the widget that had + the initial focus. + + +.. class:: SimpleDialog(master, text='', buttons=[], default=None, cancel=None, title=None, class_=None, *, bitmap=None, detail='', use_ttk=True) + + A simple modal dialog that displays the message *text* above a row of push + buttons given by *buttons*, and returns the index of the button the user + presses. + Each entry of *buttons* is either a button label, or a mapping of button + options such as ``{'text': 'OK', 'underline': 0}``; an ``underline`` option + makes :kbd:`Alt` plus the underlined character invoke the button. + *default* is the index of the default button, activated by the Return key + when no button has the focus, *cancel* the index returned when the window is + closed through the window manager, *title* the window title, and *class_* + the Tk class name of the window. + *bitmap* is the name of a bitmap displayed beside the message + (for example ``'warning'`` or ``'question'``); the standard names + ``'error'``, ``'info'``, ``'question'`` and ``'warning'`` are shown as + themed icons when *use_ttk* is true. + *detail* is a secondary message displayed below *text*. + When *use_ttk* is true (the default), the dialog is built from the themed + :mod:`tkinter.ttk` widgets, modelled on the Tk message box; when false, from + the classic :mod:`tkinter` widgets, modelled on ``tk_dialog``. + + .. versionchanged:: next + The dialog is now built from the themed :mod:`tkinter.ttk` widgets by + default, instead of the classic :mod:`tkinter` widgets. + Added the *bitmap*, *detail* and *use_ttk* parameters. + Entries of *buttons* may be mappings of button options. + + .. method:: go() + + Display the dialog, wait until the user presses a button or closes the + window, and return the index of the chosen button. + :mod:`!tkinter.filedialog` --- File selection dialogs @@ -51,7 +131,7 @@ functions for creating simple modal dialogs to get a value from the user. The :mod:`!tkinter.filedialog` module provides classes and factory functions for creating file/directory selection windows. -Native Load/Save Dialogs +Native load/save dialogs ------------------------ The following classes and functions provide file dialog windows that combine a @@ -77,34 +157,46 @@ listed below: **Static factory functions** The below functions when called create a modal, native look-and-feel dialog, -wait for the user's selection, then return the selected value(s) or ``None`` to the -caller. +wait for the user's selection, and return it. +The exact return value depends on the function (see below); when the dialog is +cancelled it is an empty string, an empty tuple, an empty list or ``None``. .. function:: askopenfile(mode="r", **options) askopenfiles(mode="r", **options) - The above two functions create an :class:`Open` dialog and return the opened - file object(s) in read-only mode. + Create an :class:`Open` dialog. + :func:`askopenfile` returns the opened file object, or ``None`` if the + dialog is cancelled. + :func:`askopenfiles` returns a list of the opened file objects, or an empty + list if cancelled. + The files are opened in mode *mode* (read-only ``'r'`` by default). .. function:: asksaveasfile(mode="w", **options) - Create a :class:`SaveAs` dialog and return a file object opened in write-only mode. + Create a :class:`SaveAs` dialog and return the opened file object, or + ``None`` if the dialog is cancelled. + The file is opened in mode *mode* (``'w'`` by default). .. function:: askopenfilename(**options) askopenfilenames(**options) - The above two functions create an :class:`Open` dialog and return the - selected filename(s) that correspond to existing file(s). + Create an :class:`Open` dialog. + :func:`askopenfilename` returns the selected filename as a string, or an + empty string if the dialog is cancelled. + :func:`askopenfilenames` returns a tuple of the selected filenames, or an + empty tuple if cancelled. .. function:: asksaveasfilename(**options) - Create a :class:`SaveAs` dialog and return the selected filename. + Create a :class:`SaveAs` dialog and return the selected filename as a + string, or an empty string if the dialog is cancelled. .. function:: askdirectory(**options) - | Prompt user to select a directory. - | Additional keyword option: - | *mustexist* - determines if selection must be an existing directory. + Prompt the user to select a directory, and return its path as a string, or + an empty string if the dialog is cancelled. + Additional keyword option: *mustexist* - if true, the user may only select + an existing directory (false by default). .. class:: Open(master=None, **options) SaveAs(master=None, **options) @@ -168,6 +260,13 @@ These do not emulate the native look-and-feel of the platform. Exit dialog returning current selection. + .. method:: ok_command() + + Called when the user confirms the current selection. + The base implementation accepts the selection and closes the dialog; + :class:`LoadFileDialog` and :class:`SaveFileDialog` override it to check + the selection first. + .. method:: quit(how=None) Exit dialog returning filename, if any. @@ -222,6 +321,40 @@ is the base class for dialogs defined in other supporting modules. Render the Dialog window. +:mod:`!tkinter.dialog` --- Classic Tk dialog boxes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: tkinter.dialog + :synopsis: A simple dialog box built on the classic Tk widgets. + +**Source code:** :source:`Lib/tkinter/dialog.py` + +-------------- + +The :mod:`!tkinter.dialog` module provides a simple modal dialog box built on +the classic (non-themed) Tk widgets. + +.. data:: DIALOG_ICON + + The name of the default bitmap (``'questhead'``) displayed by a + :class:`Dialog`. + +.. class:: Dialog(master=None, cnf={}, **kw) + + Display a modal dialog box built from the classic (non-themed) Tk widgets + and wait for the user to press one of its buttons. + The options, given through *cnf* or as keyword arguments, include *title* + (the window title), *text* (the message), *bitmap* (an icon, + :data:`DIALOG_ICON` by default), *default* (the index of the default button) + and *strings* (the sequence of button labels). + After construction, the :attr:`!num` attribute holds the index of the button + the user pressed. + + .. method:: destroy() + + Destroy the dialog window. + + .. seealso:: Modules :mod:`tkinter.messagebox`, :ref:`tut-files` diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 8b812c173b59536..25edb40e35a630a 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -724,7 +724,7 @@ Finally, we compare the two: >>> result = list(d.compare(text1, text2)) -``result`` is a list of strings, so let's pretty-print it: +``result`` is a list of strings, so let's pretty-print it:: >>> from pprint import pprint >>> pprint(result) @@ -739,7 +739,7 @@ Finally, we compare the two: '? ++++ ^ ^\n', '+ 5. Flat is better than nested.\n'] -As a single multi-line string it looks like this: +As a single multi-line string it looks like this:: >>> import sys >>> sys.stdout.writelines(result) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 3e7ae509fedcea6..4e1ba4327faec49 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -139,7 +139,7 @@ code. .. class:: Bytecode(x, *, first_line=None, current_offset=None,\ show_caches=False, adaptive=False, show_offsets=False,\ - show_positions=False) + show_positions=False, show_jit=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -170,6 +170,9 @@ code. If *show_positions* is ``True``, :meth:`.dis` will include instruction source code positions in the output. + If *show_jit* is ``True``, :meth:`.dis` will show ``ENTER_EXECUTOR`` + instructions, which mark JIT entry points and are hidden by default. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -205,6 +208,9 @@ code. .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + Example: .. doctest:: @@ -259,7 +265,8 @@ operation is being performed, so the intermediate analysis object isn't useful: .. function:: dis(x=None, *, file=None, depth=None, show_caches=False,\ - adaptive=False, show_offsets=False, show_positions=False) + adaptive=False, show_offsets=False, show_positions=False,\ + show_jit=False) Disassemble the *x* object. *x* can denote either a module, a class, a method, a function, a generator, an asynchronous generator, a coroutine, @@ -286,6 +293,9 @@ operation is being performed, so the intermediate analysis object isn't useful: If *adaptive* is ``True``, this function will display specialized bytecode that may be different from the original bytecode. + If *show_jit* is ``True``, this function will show ``ENTER_EXECUTOR`` + instructions, which mark JIT entry points and are hidden by default. + .. versionchanged:: 3.4 Added *file* parameter. @@ -304,8 +314,11 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False,\ - show_offset=False, show_positions=False) + show_offsets=False, show_positions=False, show_jit=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -326,10 +339,14 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False,\ - adaptive=False, show_offsets=False, show_positions=False) + adaptive=False, show_offsets=False, show_positions=False,\ + show_jit=False) disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False,\ - show_offsets=False, show_positions=False) + show_offsets=False, show_positions=False, show_jit=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: @@ -362,7 +379,10 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. -.. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) + .. versionchanged:: next + Added the *show_jit* parameter. + +.. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False, show_jit=False) Return an iterator over the instructions in the supplied function, method, source code string or code object. @@ -377,6 +397,8 @@ operation is being performed, so the intermediate analysis object isn't useful: The *adaptive* parameter works as it does in :func:`dis`. + The *show_jit* parameter works as it does in :func:`dis`. + .. versionadded:: 3.4 .. versionchanged:: 3.11 @@ -388,6 +410,9 @@ operation is being performed, so the intermediate analysis object isn't useful: field populated (regardless of the value of *show_caches*) and it no longer generates separate items for the cache entries. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: findlinestarts(code) This generator function uses the :meth:`~codeobject.co_lines` method diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index c6924a0ac29c972..619c17c98e8d89c 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -93,9 +93,10 @@ headers. ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. ``defects`` is an empty list. The parse method should append any detected defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may + values for at least the keys ``decoded``, ``defects`` and ``parse_tree``. + ``decoded`` should be the string value for the header (that is, the header + value fully decoded to unicode). ``parse_tree`` is set to the parse tree obtained + from parsing the header. The parse method should assume that *string* may contain content-transfer-encoded parts, but should correctly handle all valid unicode characters as well so that it can parse un-encoded header values. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index be7f59b0fce2a5e..559a76bb3963e09 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -115,30 +115,30 @@ Module contents :class:`StrEnum` defaults to the lower-cased version of the member name, while other Enums default to 1 and increase from there. - :func:`~enum.property` + :deco:`~enum.property` Allows :class:`Enum` members to have attributes without conflicting with member names. The ``value`` and ``name`` attributes are implemented this way. - :func:`unique` + :deco:`unique` Enum class decorator that ensures only one name is bound to any one value. - :func:`verify` + :deco:`verify` Enum class decorator that checks user-selectable constraints on an enumeration. - :func:`member` + :deco:`member` Make ``obj`` a member. Can be used as a decorator. - :func:`nonmember` + :deco:`nonmember` Do not make ``obj`` a member. Can be used as a decorator. - :func:`global_enum` + :deco:`global_enum` Modify the :class:`str() ` and :func:`repr` of an enum to show its members as belonging to the module instead of its class, @@ -222,7 +222,7 @@ Data types names of the members in *cls*:: >>> dir(Color) - ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__'] + ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__', '_generate_next_value_', '_missing_'] .. method:: EnumType.__getitem__(cls, name) @@ -355,7 +355,7 @@ Data types ... print(f'today is {cls(dt.date.today().isoweekday()).name}') ... >>> dir(Weekday.SATURDAY) - ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] + ['__class__', '__doc__', '__eq__', '__hash__', '__module__', '_add_alias_', '_add_value_alias_', '_generate_next_value_', '_missing_', 'name', 'today', 'value'] .. method:: Enum._generate_next_value_(name, start, count, last_values) @@ -629,7 +629,7 @@ Data types >>> white in purple False - .. method:: __iter__(self): + .. method:: __iter__(self) Returns all contained non-alias members:: @@ -640,7 +640,7 @@ Data types .. versionadded:: 3.11 - .. method:: __len__(self): + .. method:: __len__(self) Returns number of members in flag:: @@ -651,7 +651,7 @@ Data types .. versionadded:: 3.11 - .. method:: __bool__(self): + .. method:: __bool__(self) Returns *True* if any members in flag, *False* otherwise:: @@ -688,7 +688,7 @@ Data types >>> purple ^ Color.GREEN - .. method:: __invert__(self): + .. method:: __invert__(self) Returns all the flags in *type(self)* that are not in *self*:: @@ -1006,7 +1006,7 @@ Utilities and decorators .. decorator:: property - A decorator similar to the built-in *property*, but specifically for + A decorator similar to the built-in :deco:`property`, but specifically for enumerations. It allows member attributes to have the same names as members themselves. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 7fc6055aa9a8812..3775d5ac81a2736 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -984,6 +984,9 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + Exception groups are :ref:`generic ` over the type of their + contained exceptions. + .. impl-detail:: The ``excs`` parameter may be any sequence, but lists and tuples are diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 529e97bae6df8e2..905e1acd433617d 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -67,7 +67,7 @@ Dumping the traceback .. versionchanged:: 3.5 Added support for passing file descriptor to this function. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *max_threads* keyword argument. @@ -140,7 +140,7 @@ Fault handler state .. versionchanged:: 3.14 The dump now displays the C stack trace if *c_stack* is true. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *max_threads* keyword argument. .. function:: disable() @@ -178,7 +178,7 @@ Dumping the tracebacks after a timeout .. versionchanged:: 3.7 This function is now always available. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *max_threads* keyword argument. .. function:: cancel_dump_traceback_later() @@ -204,7 +204,7 @@ Dumping the traceback on a user signal .. versionchanged:: 3.5 Added support for passing file descriptor to this function. - .. versionchanged:: next + .. versionchanged:: 3.15 Added the *max_threads* keyword argument. .. function:: unregister(signum) diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index a213679e4e2dd76..fc99b7fd1e7523a 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -51,7 +51,7 @@ Unless stated otherwise, "filename string" and "pattern string" either refer to functions documented below do not allow to mix a :class:`!bytes` pattern with a :class:`!str` filename, and vice-versa. -Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 +Finally, note that :deco:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 06fd5cdc7be2a63..a4d37beb436899a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -19,24 +19,25 @@ are always available. They are listed here in alphabetical order. | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`sentinel` | -| | :func:`bool` | | | | | | :func:`slice` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`sorted` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | :func:`staticmethod` | -| | |func-bytes|_ | | :func:`globals` | | | | |func-str|_ | -| | | | | | **O** | | :func:`sum` | -| | **C** | | **H** | | :func:`object` | | :func:`super` | -| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | :func:`compile` | | :func:`hex` | | | | | -| | :func:`complex` | | | | **P** | | **V** | -| | | | **I** | | :func:`pow` | | :func:`vars` | -| | **D** | | :func:`id` | | :func:`print` | | | -| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | -| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | -| | :func:`dir` | | :func:`isinstance` | | | | | -| | :func:`divmod` | | :func:`issubclass` | | | | **_** | +| | :func:`bin` | | |func-frozendict|_ | | :func:`min` | | :func:`sentinel` | +| | :func:`bool` | | |func-frozenset|_ | | | | :func:`slice` | +| | :func:`breakpoint` | | | | **N** | | :func:`sorted` | +| | |func-bytearray|_ | | **G** | | :func:`next` | | :func:`staticmethod` | +| | |func-bytes|_ | | :func:`getattr` | | | | |func-str|_ | +| | | | :func:`globals` | | **O** | | :func:`sum` | +| | **C** | | | | :func:`object` | | :func:`super` | +| | :func:`callable` | | **H** | | :func:`oct` | | | +| | :func:`chr` | | :func:`hasattr` | | :func:`open` | | **T** | +| | :func:`classmethod` | | :func:`hash` | | :func:`ord` | | |func-tuple|_ | +| | :func:`compile` | | :func:`help` | | | | :func:`type` | +| | :func:`complex` | | :func:`hex` | | **P** | | | +| | | | | | :func:`pow` | | **V** | +| | **D** | | **I** | | :func:`print` | | :func:`vars` | +| | :func:`delattr` | | :func:`id` | | :func:`property` | | | +| | |func-dict|_ | | :func:`input` | | | | **Z** | +| | :func:`dir` | | :func:`int` | | | | :func:`zip` | +| | :func:`divmod` | | :func:`isinstance` | | | | | +| | | | :func:`issubclass` | | | | **_** | | | | | :func:`iter` | | | | :func:`__import__` | +-------------------------+-----------------------+-----------------------+-------------------------+ @@ -44,6 +45,7 @@ are always available. They are listed here in alphabetical order. used, with replacement texts to make the output in the table consistent .. |func-dict| replace:: ``dict()`` +.. |func-frozendict| replace:: ``frozendict()`` .. |func-frozenset| replace:: ``frozenset()`` .. |func-memoryview| replace:: ``memoryview()`` .. |func-set| replace:: ``set()`` @@ -483,10 +485,10 @@ are always available. They are listed here in alphabetical order. :noindex: Create a new dictionary. The :class:`dict` object is the dictionary class. - See :class:`dict` and :ref:`typesmapping` for documentation about this class. + See also :ref:`typesmapping` for documentation about this class. - For other containers see the built-in :class:`list`, :class:`set`, and - :class:`tuple` classes, as well as the :mod:`collections` module. + For other containers see the built-in :class:`frozendict`, :class:`list`, + :class:`set`, and :class:`tuple` classes, as well as the :mod:`collections` module. .. function:: dir() @@ -864,12 +866,27 @@ are always available. They are listed here in alphabetical order. if *format_spec* is not an empty string. +.. _func-frozendict: +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + :noindex: + + Create a new frozen dictionary. The :class:`frozendict` object is a built-in class. + See also :ref:`typesmapping` for documentation about this class. + + For other containers see the built-in :class:`dict`, :class:`list`, :class:`set`, + and :class:`tuple` classes, as well as the :mod:`collections` module. + + .. versionadded:: 3.15 + + .. _func-frozenset: .. class:: frozenset(iterable=(), /) :noindex: Return a new :class:`frozenset` object, optionally with elements taken from - *iterable*. ``frozenset`` is a built-in class. See :class:`frozenset` and + *iterable*. :class:`frozenset` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`set`, :class:`list`, @@ -1107,13 +1124,13 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: issubclass(class, classinfo, /) +.. function:: issubclass(cls, classinfo, /) - Return ``True`` if *class* is a subclass (direct, indirect, or :term:`virtual + Return ``True`` if *cls* is a subclass (direct, indirect, or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class objects (or recursively, other such tuples) - or a :ref:`types-union`, in which case return ``True`` if *class* is a + or a :ref:`types-union`, in which case return ``True`` if *cls* is a subclass of any entry in *classinfo*. In any other case, a :exc:`TypeError` exception is raised. @@ -1663,7 +1680,7 @@ are always available. They are listed here in alphabetical order. If given, *doc* will be the docstring of the property attribute. Otherwise, the property will copy *fget*'s docstring (if it exists). This makes it possible to - create read-only properties easily using :func:`property` as a :term:`decorator`:: + create read-only properties easily using :deco:`property` as a :term:`decorator`:: class Parrot: def __init__(self): @@ -1797,7 +1814,7 @@ are always available. They are listed here in alphabetical order. :noindex: Return a new :class:`set` object, optionally with elements taken from - *iterable*. ``set`` is a built-in class. See :class:`set` and + *iterable*. :class:`set` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`frozenset`, :class:`list`, @@ -1827,15 +1844,21 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. -.. class:: sentinel(name, /) +.. class:: sentinel(name, /, *, repr=None) Return a new unique sentinel object. *name* must be a :class:`str`, and is - used as the returned object's representation:: + used by default as the returned object's representation:: >>> MISSING = sentinel("MISSING") >>> MISSING MISSING + The optional *repr* argument can be used to specify a different representation:: + + >>> MISSING = sentinel("MISSING", repr="") + >>> MISSING + + Sentinel objects are truthy and compare equal only to themselves. They are intended to be compared with the :keyword:`is` operator. @@ -1879,9 +1902,9 @@ are always available. They are listed here in alphabetical order. .. attribute:: __module__ - The name of the module where the sentinel was created. + The name of the module where the sentinel was created. This attribute is writable. - .. versionadded:: next + .. versionadded:: 3.15 .. class:: slice(stop, /) @@ -1961,7 +1984,7 @@ are always available. They are listed here in alphabetical order. be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see - :func:`classmethod` for a variant that is useful for creating alternate class + :deco:`classmethod` for a variant that is useful for creating alternate class constructors. Like all decorators, it is also possible to call ``staticmethod`` as diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 7da59cba5170b35..2b46978f058102b 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -27,7 +27,7 @@ The :mod:`!functools` module defines the following functions: Returns the same as ``lru_cache(maxsize=None)``, creating a thin wrapper around a dictionary lookup for the function arguments. Because it never needs to evict old values, this is smaller and faster than - :func:`lru_cache` with a size limit. + :deco:`lru_cache` with a size limit. For example:: @@ -61,7 +61,7 @@ The :mod:`!functools` module defines the following functions: Transform a method of a class into a property whose value is computed once and then cached as a normal attribute for the life of the instance. Similar - to :func:`property`, with the addition of caching. Useful for expensive + to :deco:`property`, with the addition of caching. Useful for expensive computed properties of instances that are otherwise effectively immutable. Example:: @@ -75,8 +75,8 @@ The :mod:`!functools` module defines the following functions: def stdev(self): return statistics.stdev(self._data) - The mechanics of :func:`cached_property` are somewhat different from - :func:`property`. A regular property blocks attribute writes unless a + The mechanics of :deco:`cached_property` are somewhat different from + :deco:`property`. A regular property blocks attribute writes unless a setter is defined. In contrast, a *cached_property* allows writes. The *cached_property* decorator only runs on lookups and only when an @@ -108,14 +108,14 @@ The :mod:`!functools` module defines the following functions: (as such classes don't provide a ``__dict__`` attribute at all). If a mutable mapping is not available or if space-efficient key sharing is - desired, an effect similar to :func:`cached_property` can also be achieved by - stacking :func:`property` on top of :func:`lru_cache`. See - :ref:`faq-cache-method-calls` for more details on how this differs from :func:`cached_property`. + desired, an effect similar to :deco:`cached_property` can also be achieved by + stacking :deco:`property` on top of :deco:`lru_cache`. See + :ref:`faq-cache-method-calls` for more details on how this differs from :deco:`cached_property`. .. versionadded:: 3.8 .. versionchanged:: 3.12 - Prior to Python 3.12, ``cached_property`` included an undocumented lock to + Prior to Python 3.12, :deco:`!cached_property` included an undocumented lock to ensure that in multi-threaded usage the getter function was guaranteed to run only once per instance. However, the lock was per-property, not per-instance, which could result in unacceptably high lock contention. In @@ -732,7 +732,7 @@ The :mod:`!functools` module defines the following functions: function's :attr:`~function.__dict__`, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes - (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function + (e.g. bypassing a caching decorator such as :deco:`lru_cache`), this function automatically adds a ``__wrapped__`` attribute to the wrapper that refers to the function being wrapped. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 701af579453ce34..65533e7e57adc33 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -133,11 +133,6 @@ The :mod:`!gc` module provides the following functions: With the third generation, things are a bit more complicated, see `Collecting the oldest generation `_ for more information. - In the free-threaded build, the increase in process memory usage is also - checked before running the collector. If the memory usage has not increased - by 10% since the last collection and the net number of object allocations - has not exceeded 40 times *threshold0*, the collection is not run. - See `Garbage collector design `_ for more information. .. versionchanged:: 3.14 diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index fd96f3bbf6a574f..f60cc63414f764d 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -18,7 +18,7 @@ The :mod:`!getpass` module provides two functions: the string *prompt*, which defaults to ``'Password: '``. On Unix, the prompt is written to the file-like object *stream* using the replace error handler if needed. *stream* defaults to the controlling terminal - (:file:`/dev/tty`) or if that is unavailable to ``sys.stderr`` (this + (:file:`/dev/tty`) or if that is unavailable to :data:`sys.stderr` (this argument is ignored on Windows). The *echo_char* argument controls how user input is displayed while typing. @@ -27,12 +27,12 @@ The :mod:`!getpass` module provides two functions: typed character is replaced by it. For example, ``echo_char='*'`` will display asterisks instead of the actual input. - If echo free input is unavailable getpass() falls back to printing - a warning message to *stream* and reading from ``sys.stdin`` and + If echo-free input is unavailable, :func:`getpass` falls back to printing + a warning message to *stream* and reading from :data:`sys.stdin` and issuing a :exc:`GetPassWarning`. .. note:: - If you call getpass from within IDLE, the input may be done in the + If you call :func:`getpass` from within IDLE, the input may be done in the terminal you launched IDLE from rather than the idle window itself. .. note:: diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 2de16fe40362b3e..2ab7ba7df19cf14 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -51,19 +51,19 @@ class-based API instead. .. index:: single: _ (underscore); gettext -.. function:: gettext(message) +.. function:: gettext(message, /) Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as :func:`!_` in the local namespace (see examples below). -.. function:: dgettext(domain, message) +.. function:: dgettext(domain, message, /) Like :func:`.gettext`, but look the message up in the specified *domain*. -.. function:: ngettext(singular, plural, n) +.. function:: ngettext(singular, plural, n, /) Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some @@ -78,15 +78,15 @@ class-based API instead. formulas for a variety of languages. -.. function:: dngettext(domain, singular, plural, n) +.. function:: dngettext(domain, singular, plural, n, /) Like :func:`ngettext`, but look the message up in the specified *domain*. -.. function:: pgettext(context, message) -.. function:: dpgettext(domain, context, message) -.. function:: npgettext(context, singular, plural, n) -.. function:: dnpgettext(domain, context, singular, plural, n) +.. function:: pgettext(context, message, /) +.. function:: dpgettext(domain, context, message, /) +.. function:: npgettext(context, singular, plural, n, /) +.. function:: dnpgettext(domain, context, singular, plural, n, /) Similar to the corresponding functions without the ``p`` in the prefix (that is, :func:`gettext`, :func:`dgettext`, :func:`ngettext`, :func:`dngettext`), @@ -223,20 +223,20 @@ are the methods of :class:`!NullTranslations`: translation for a given message. - .. method:: gettext(message) + .. method:: gettext(message, /) If a fallback has been set, forward :meth:`!gettext` to the fallback. Otherwise, return *message*. Overridden in derived classes. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) If a fallback has been set, forward :meth:`!ngettext` to the fallback. Otherwise, return *singular* if *n* is 1; return *plural* otherwise. Overridden in derived classes. - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) If a fallback has been set, forward :meth:`pgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -244,7 +244,7 @@ are the methods of :class:`!NullTranslations`: .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) If a fallback has been set, forward :meth:`npgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -322,7 +322,7 @@ unexpected, or if other problems occur while reading the file, instantiating a The following methods are overridden from the base class implementation: - .. method:: gettext(message) + .. method:: gettext(message, /) Look up the *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no entry in the catalog for the @@ -331,7 +331,7 @@ unexpected, or if other problems occur while reading the file, instantiating a *message* id is returned. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to determine which @@ -352,7 +352,7 @@ unexpected, or if other problems occur while reading the file, instantiating a n) % {'num': n} - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) Look up the *context* and *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no @@ -363,7 +363,7 @@ unexpected, or if other problems occur while reading the file, instantiating a .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index ed9fdaf1d727b08..9211e5f18c6b6e6 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -28,7 +28,7 @@ Note that additional file formats which can be decompressed by the The module defines the following items: -.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None, *, mtime=None) Open a gzip-compressed file in binary or text mode, returning a :term:`file object`. @@ -43,9 +43,12 @@ The module defines the following items: The *compresslevel* argument is an integer from 0 to 9, as for the :class:`GzipFile` constructor. + The keyword-only argument *mtime* represents a Unix timestamp. + For binary mode, this function is equivalent to the :class:`GzipFile` - constructor: ``GzipFile(filename, mode, compresslevel)``. In this case, the - *encoding*, *errors* and *newline* arguments must not be provided. + constructor: ``GzipFile(filename, mode, compresslevel, mtime=mtime)``. + In this case, the *encoding*, *errors* and *newline* arguments must not + be provided. For text mode, a :class:`GzipFile` object is created, and wrapped in an :class:`io.TextIOWrapper` instance with the specified encoding, error @@ -66,6 +69,10 @@ The module defines the following items: It is the default level used by most compression tools and a better tradeoff between speed and performance. + .. versionchanged:: next + Added keyword-only argument *mtime* which is passed to the class + constructor of :class:`~gzip.GzipFile`. + .. exception:: BadGzipFile An exception raised for invalid gzip files. It inherits from :exc:`OSError`. @@ -108,9 +115,13 @@ The module defines the following items: is no compression. The default is ``9``. The optional *mtime* argument is the timestamp requested by gzip. The time - is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 - to generate a compressed stream that does not depend on creation time. + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. Set + *mtime* to ``0`` to generate a compressed stream that does not depend on + creation time. If *mtime* is omitted or ``None``, the current time is used; + however, if the current time is outside the range 00:00:00 UTC, January 1, + 1970 through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* + argument is outside the range ``0`` to ``2**32-1``, then the value ``0`` + is used instead. See below for the :attr:`mtime` attribute that is set when decompressing. diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 772f2633b294431..c4b9173f9e34eb2 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -379,7 +379,7 @@ instantiation, of which this module provides three different variants: .. versionchanged:: 3.9 The *directory* parameter accepts a :term:`path-like object`. - .. versionchanged:: next + .. versionchanged:: 3.15 Added *extra_response_headers* parameter. A lot of the work, such as parsing the request, is done by the base class @@ -400,7 +400,7 @@ instantiation, of which this module provides three different variants: cannot be guessed from the file extension of the requested URL. By default, it is set to ``'application/octet-stream'``. - .. versionadded:: next + .. versionadded:: 3.15 .. attribute:: extensions_map @@ -562,7 +562,7 @@ The following options are accepted: python -m http.server --content-type text/html - .. versionadded:: next + .. versionadded:: 3.15 .. option:: --tls-cert @@ -604,7 +604,7 @@ The following options are accepted: headers. Headers that are sent automatically by the server (for instance Content-Type) will not be overwritten by the server. - .. versionadded:: next + .. versionadded:: 3.15 .. _http.server-security: diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index b29b02d3cf5fe8b..df2468f7124e6d6 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -199,6 +199,11 @@ An :class:`IMAP4` instance has the following methods: Append *message* to named mailbox. + *flags* may be ``None`` or a string of IMAP flag tokens. Multiple + flags are separated by spaces, for example ``r'\Seen \Answered'``. + If *flags* is not already enclosed in parentheses, parentheses are + added automatically. + .. method:: IMAP4.authenticate(mechanism, authobject) @@ -695,6 +700,16 @@ The following attributes are defined on instances of :class:`IMAP4`: .. versionadded:: 3.5 +.. property:: IMAP4.file + + Internal :class:`~io.BufferedReader` associated with the underlying socket. + This property is documented for legacy purposes but not part of the public + interface. The caller is responsible to ensure that the current file is + closed before changing it. + + .. deprecated-removed:: 3.15 3.19 + + .. _imap4-example: IMAP4 Example diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 63de4f91f4ba5f5..e11db37b9fad501 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -105,6 +105,13 @@ You can also get a :ref:`distribution's version number `, list its current Python environment. +.. exception:: MetadataNotFound + + Subclass of :class:`FileNotFoundError` raised when attempting to load metadata + from a distribution folder that is empty or otherwise does not contain a + metadata file. + + Functional API ============== @@ -224,6 +231,9 @@ Distribution metadata Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. + Raises :exc:`MetadataNotFound` if a distribution package is + present but no METADATA file is present. + .. class:: PackageMetadata A concrete implementation of the @@ -252,6 +262,12 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: The full set of available metadata is not described here. See the PyPA `Core metadata specification `_ for additional details. +.. versionchanged:: 3.15 + Previously and incidentally, if a METADATA file was missing from a distribution, an + empty ``PackageMetadata`` would be returned, indistinguishable from + an empty METADATA file. Now, a missing METADATA file triggers a + ``MetadataNotFound`` exception. + .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. @@ -465,6 +481,9 @@ The same applies for :func:`entry_points` and :func:`files`. .. attribute:: metadata :type: PackageMetadata + Raises :exc:`MetadataNotFound` if the METADATA file is not present in + the distribution. + There are all kinds of additional metadata available on :class:`!Distribution` instances as a :class:`PackageMetadata` instance:: diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 653fa61420be869..72db66f9f06f890 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -240,7 +240,6 @@ For all the following functions: .. versionchanged:: 3.13 Multiple *path_names* are accepted. - *encoding* and *errors* must be given as keyword arguments. .. function:: is_resource(anchor, *path_names) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 4825ac11ae2ee37..a0f7379b12a8a62 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -416,17 +416,47 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) Return ``True`` if the object is a bound method written in Python. + .. note:: -.. function:: ispackage(object) + For example, given this class:: - Return ``True`` if the object is a :term:`package`. + >>> class Greeter: + ... def say_hello(self): + ... print('hello!') - .. versionadded:: 3.14 + A bound method (also known as an *instance method*) is created when + accessing ``say_hello`` (a :term:`function` defined in the + ``Greeter`` namespace) through an instance of the ``Greeter`` class:: + + >>> instance = Greeter() + + >>> instance.say_hello + > + >>> ismethod(instance.say_hello) + True + >>> isfunction(instance.say_hello) + False + + Accessing ``say_hello`` through the ``Greeter`` class will return the + function itself. For this function, :func:`ismethod` will return + ``False``, but :func:`isfunction` will return ``True``:: + + >>> Greeter.say_hello + + >>> ismethod(Greeter.say_hello) + False + >>> isfunction(Greeter.say_hello) + True + + See :ref:`typesmethods` for details. .. function:: isfunction(object) @@ -434,11 +464,23 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a Python function, which includes functions created by a :term:`lambda` expression. + See the note for :func:`~inspect.ismethod` for an example. + + +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + .. function:: isgeneratorfunction(object) Return ``True`` if the object is a Python generator function. + It also returns ``True`` for bound methods created from Python generator functions + (see :ref:`typesmethods` for more information). + .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. @@ -1189,7 +1231,7 @@ Classes and functions times. -.. function:: getfullargspec(func) +.. function:: getfullargspec(func, *, annotation_format=Format.VALUE) Get the names and default values of a Python function's parameters. A :term:`named tuple` is returned: @@ -1219,6 +1261,14 @@ Classes and functions APIs. This function is retained primarily for use in code that needs to maintain compatibility with the Python 2 ``inspect`` module API. + A member of the + :class:`annotationlib.Format` enum can be passed to the + *annotation_format* parameter to control the format of the returned + annotations. For example, use + ``annotation_format=annotationlib.Format.STRING`` to return annotations in string + format. Note that with the default ``VALUE`` format, creation of some argspecs + may raise an exception. + .. versionchanged:: 3.4 This function is now based on :func:`signature`, but still ignores ``__wrapped__`` attributes and includes the already bound first @@ -1236,6 +1286,9 @@ Classes and functions order of keyword-only parameters as of version 3.7, although in practice this order had always been preserved in Python 3. + .. versionchanged:: 3.15 + The *annotation_format* parameter was added. + .. function:: getargvalues(frame) @@ -1561,10 +1614,11 @@ properties, will be invoked and :meth:`~object.__getattr__` and may be called. For cases where you want passive introspection, like documentation tools, this -can be inconvenient. :func:`getattr_static` has the same signature as :func:`getattr` +can be inconvenient. :func:`getattr_static` has a similar signature as :func:`getattr` but avoids executing code when it fetches attributes. -.. function:: getattr_static(obj, attr, default=None) +.. function:: getattr_static(obj, attr) + getattr_static(obj, attr, default) Retrieve attributes without triggering dynamic lookup via the descriptor protocol, :meth:`~object.__getattr__` @@ -1839,7 +1893,7 @@ appending a colon and the qualified name of the target object. Print information about the specified object rather than the source code -.. versionchanged:: next +.. versionchanged:: 3.15 The ``--details`` option now supports basic introspection for modules without available source code and indicates when modules are frozen. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 494e57fe1c04743..d47b74efe22de9d 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -38,6 +38,7 @@ will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since :exc:`IOError` is now an alias of :exc:`OSError`. +.. _text-io: Text I/O ^^^^^^^^ @@ -65,6 +66,7 @@ In-memory text streams are also available as :class:`StringIO` objects:: The text stream API is described in detail in the documentation of :class:`TextIOBase`. +.. _binary-io: Binary I/O ^^^^^^^^^^ @@ -103,6 +105,13 @@ stream by opening a file in binary mode with buffering disabled:: The raw stream API is described in detail in the docs of :class:`RawIOBase`. +.. warning:: + Raw I/O is a low-level interface and methods generally must have their return + values checked and be explicitly retried to ensure an operation completes. + For instance :meth:`~RawIOBase.write` returns the number of bytes written + which may be less than the number of bytes provided (a partial write). + High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement + retry behavior. .. _io-text-encoding: @@ -478,8 +487,11 @@ I/O Base Classes Read up to *size* bytes from the object and return them. As a convenience, if *size* is unspecified or -1, all bytes until EOF are returned. - Otherwise, only one system call is ever made. Fewer than *size* bytes may - be returned if the operating system call returns fewer than *size* bytes. + + Attempts to make only one system call but will retry if interrupted and + the signal handler does not raise an exception (see :pep:`475` for the + rationale). This means fewer than *size* bytes may be returned if the + operating system call returns fewer than *size* bytes. If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, @@ -493,13 +505,19 @@ I/O Base Classes Read and return all the bytes from the stream until EOF, using multiple calls to the stream if necessary. + If ``0`` bytes are returned this indicates end of file. If the object is in + non-blocking mode and the underlying :meth:`read` returns ``None`` + indicating no bytes are available, ``None`` is returned. + .. method:: readinto(b, /) Read bytes into a pre-allocated, writable :term:`bytes-like object` *b*, and return the number of bytes read. For example, *b* might be a :class:`bytearray`. - If the object is in non-blocking mode and no bytes - are available, ``None`` is returned. + + If ``0`` is returned and ``len(b)`` is not ``0``, this indicates end of file. If + the object is in non-blocking mode and no bytes are available, ``None`` is + returned. .. method:: write(b, /) @@ -513,6 +531,13 @@ I/O Base Classes this method returns, so the implementation should only access *b* during the method call. + .. warning:: + + This function does not ensure all bytes are written or an exception is + thrown. Callers may implement that behavior by checking the return + value and, if it is less than the length of *b*, looping with additional + write calls until all unwritten bytes are written. High-level I/O + objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior. .. class:: BufferedIOBase @@ -641,7 +666,11 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits from :class:`RawIOBase`. + inherits from :class:`RawIOBase` and implements its low-level access design. + This means :meth:`~RawIOBase.write` does not guarantee all bytes are written + and :meth:`~RawIOBase.read` may read less bytes than requested even when more + bytes may be present in the underlying file. To get "write all" and + "read at least" behavior, use :ref:`binary-io`. The *name* can be one of two things: @@ -661,10 +690,6 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`~RawIOBase.read` (when called with a positive argument), - :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this - class will only make one system call. - A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with (*name*, *flags*). *opener* must return an open file descriptor (passing @@ -676,6 +701,13 @@ Raw File I/O See the :func:`open` built-in function for examples on using the *opener* parameter. + .. warning:: + :class:`FileIO` is a low-level I/O object and members, such as + :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their + return values checked explicitly in a retry loop to implement "write all" + and "read at least" behavior. High-level I/O objects :ref:`binary-io` and + :ref:`text-io` implement retry behavior. + .. versionchanged:: 3.3 The *opener* parameter was added. The ``'x'`` mode was added. @@ -824,9 +856,9 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) - A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` - and :class:`BufferedWriter`. + A buffered binary stream implementing :class:`BufferedIOBase` interfaces + providing higher-level access to a seekable :class:`RawIOBase` raw binary + stream. The constructor creates a reader and writer for a seekable raw stream, given in the first argument. If the *buffer_size* is omitted it defaults to diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b354e7ba534835f..383ccad9df041b5 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -211,7 +211,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 30bf7860a751192..3842c1e9d477d29 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -124,12 +124,12 @@ in :mod:`logging` itself) and defining handlers which are declared either in application (e.g. based on command-line parameters or other aspects of the runtime environment) before being passed to ``fileConfig``. - .. versionchanged:: 3.10 - Added the *encoding* parameter. + .. versionchanged:: 3.10 + Added the *encoding* parameter. - .. versionchanged:: 3.12 - An exception will be thrown if the provided file - doesn't exist or is invalid or empty. + .. versionchanged:: 3.12 + An exception will be thrown if the provided file + doesn't exist or is invalid or empty. .. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 714db5fa12af0aa..5152c7561fa1f26 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -402,7 +402,8 @@ timed intervals. rollover interval. When computing the next rollover time for the first time (when the handler - is created), the last modification time of an existing log file, or else + is created), the creation time (if supported by the OS and file system) + or the last modification of an existing log file, or else the current time, is used to compute when the next rotation will occur. If the *utc* argument is true, times in UTC will be used; otherwise @@ -449,6 +450,10 @@ timed intervals. .. versionchanged:: 3.9 The *errors* parameter was added. + .. versionchanged:: next + Use the creation time instead of the last modification time, if supported by the OS and file system. + + .. method:: doRollover() Does a rollover, as described above. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index aba530844d7177a..a3d117c10702414 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -690,7 +690,7 @@ Formatter Objects :param defaults: A dictionary with default values to use in custom fields. For example, ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` - :type defaults: dict[str, Any] + :type defaults: dict[str, typing.Any] .. versionchanged:: 3.2 Added the *style* parameter. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 6cede00b2186789..cd72174d54f6e62 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -356,12 +356,26 @@ options. Valid filter IDs are as follows: * Branch-Call-Jump (BCJ) filters: - * :const:`FILTER_X86` - * :const:`FILTER_IA64` - * :const:`FILTER_ARM` - * :const:`FILTER_ARMTHUMB` - * :const:`FILTER_POWERPC` - * :const:`FILTER_SPARC` + * :const:`!FILTER_X86` + * :const:`!FILTER_IA64` + * :const:`!FILTER_ARM` + * :const:`!FILTER_ARMTHUMB` + * :const:`!FILTER_POWERPC` + * :const:`!FILTER_SPARC` + + The above work on all lzma runtime library versions. + + * :const:`!FILTER_ARM64` + + Only works if the lzma version is 5.4.0 or later. + + .. versionadded:: next + + * :const:`!FILTER_RISCV` + + Only works if the lzma version is 5.6.0 or later. + + .. versionadded:: next A filter chain can consist of up to 4 filters, and cannot be empty. The last filter in the chain must be a compression filter, and any other filters must be diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 9cc8c5d6886324c..efe411e5a43f27d 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -83,13 +83,20 @@ noted otherwise, all return values are floats. **Trigonometric functions** -------------------------------------------------------------------------------------------------- -:func:`acos(x) ` Arc cosine of *x* -:func:`asin(x) ` Arc sine of *x* -:func:`atan(x) ` Arc tangent of *x* -:func:`atan2(y, x) ` ``atan(y / x)`` -:func:`cos(x) ` Cosine of *x* -:func:`sin(x) ` Sine of *x* -:func:`tan(x) ` Tangent of *x* +:func:`acos(x) ` Arc cosine of *x*, in radians +:func:`acospi(x) ` Arc cosine of *x*, in half-turns +:func:`asin(x) ` Arc sine of *x*, in radians +:func:`asinpi(x) ` Arc sine of *x*, in half-turns +:func:`atan(x) ` Arc tangent of *x*, in radians +:func:`atanpi(x) ` Arc tangent of *x*, in half-turns +:func:`atan2(y, x) ` ``atan(y / x)``, in radians +:func:`atan2pi(y, x) ` ``atan(y / x)``, in half-turns +:func:`cos(x) ` Cosine of *x* radians +:func:`cospi(x) ` Cosine of *x⋅π* radians +:func:`sin(x) ` Sine of *x* radians +:func:`sinpi(x) ` Sine of *x⋅π* radians +:func:`tan(x) ` Tangent of *x* radians +:func:`tanpi(x) ` Tangent of *x⋅π* radians **Hyperbolic functions** -------------------------------------------------------------------------------------------------- @@ -255,10 +262,12 @@ Floating point manipulation functions .. function:: frexp(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the mantissa and exponent of *x* as the pair ``(m, e)``. + If *x* is a finite nonzero number, then *m* is a float with + ``0.5 <= abs(m) < 1.0`` and an integer *e* is such that + ``x == m * 2**e`` exactly. Else, return ``(x, 0)``. + This is used to "pick apart" the internal representation of + a float in a portable way. Note that :func:`frexp` has a different call/return pattern than its C equivalents: it takes a single argument and return a pair of @@ -597,18 +606,42 @@ Trigonometric functions ``pi``. +.. function:: acospi(x) + + Return the arc cosine of *x*, in half-turns. The result is between ``0`` and + ``1``. + + .. versionadded:: next + + .. function:: asin(x) Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and ``pi/2``. +.. function:: asinpi(x) + + Return the arc sine of *x*, in half-turns. The result is between ``-0.5`` and + ``0.5``. + + .. versionadded:: next + + .. function:: atan(x) Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and ``pi/2``. +.. function:: atanpi(x) + + Return the arc tangent of *x*, in half-turns. The result is between ``-0.5`` and + ``0.5``. + + .. versionadded:: next + + .. function:: atan2(y, x) Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. @@ -619,21 +652,54 @@ Trigonometric functions -1)`` is ``-3*pi/4``. +.. function:: atan2pi(y, x) + + Return ``atanpi(y / x)``, in half-turns. The result is between ``-1`` and ``1``. + The vector in the plane from the origin to point ``(x, y)`` makes this angle + with the positive X axis. The point of :func:`atan2pi` is that the signs of both + inputs are known to it, so it can compute the correct quadrant for the angle. + For example, ``atanpi(1)`` and ``atan2pi(1, 1)`` are both ``0.25``, but + ``atan2pi(-1, -1)`` is ``-0.75``. + + .. versionadded:: next + + .. function:: cos(x) Return the cosine of *x* radians. +.. function:: cospi(x) + + Return the cosine of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + .. function:: sin(x) Return the sine of *x* radians. +.. function:: sinpi(x) + + Return the sine of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + .. function:: tan(x) Return the tangent of *x* radians. +.. function:: tanpi(x) + + Return the tangent of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + Hyperbolic functions -------------------- diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 0facacd50fd389e..5c29fff146eef00 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -39,8 +39,8 @@ the information :func:`init` sets up. (e.g. :program:`compress` or :program:`gzip`). The encoding is suitable for use as a :mailheader:`Content-Encoding` header, **not** as a :mailheader:`Content-Transfer-Encoding` header. The mappings are table driven. - Encoding suffixes are case sensitive; type suffixes are first tried case - sensitively, then case insensitively. + Encoding suffixes are case-sensitive. Suffix mappings and type suffixes are + first tried case-sensitively, then case-insensitively. The optional *strict* argument is a flag specifying whether the list of known MIME types is limited to only the official types `registered with IANA @@ -129,10 +129,20 @@ behavior of the module. Add a mapping from the MIME type *type* to the extension *ext*. When the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. + Valid extensions are empty or start with a ``'.'``. + + Registered lower-case extensions are matched case-insensitively. When *strict* is ``True`` (the default), the mapping will be added to the official MIME types, otherwise to the non-standard ones. + .. deprecated:: 3.14 + *ext* values that do not start with ``'.'`` are deprecated. + + .. versionchanged:: next + *ext* now must start with ``'.'``. Otherwise :exc:`ValueError` is raised. + + .. data:: inited @@ -304,12 +314,16 @@ than one MIME-type database; it provides an interface similar to the one of the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. + Registered lower-case extensions are matched case-insensitively. + When *strict* is ``True`` (the default), the mapping will be added to the official MIME types, otherwise to the non-standard ones. - .. deprecated-removed:: 3.14 3.16 - Invalid, undotted extensions will raise a - :exc:`ValueError` in Python 3.16. + .. deprecated:: 3.14 + *ext* values that do not start with ``'.'`` are deprecated. + + .. versionchanged:: next + *ext* now must start with ``'.'``. Otherwise :exc:`ValueError` is raised. .. _mimetypes-cli: @@ -348,7 +362,7 @@ it converts file extensions to MIME types. For each ``type`` entry, the script writes a line into the standard output stream. If an unknown type occurs, it writes an error message into the -standard error stream and exits with the return code ``1``. +standard output stream and exits with the return code ``1``. .. mimetypes-cli-example: @@ -375,7 +389,7 @@ interface: $ # get a MIME type for a rare file extension $ python -m mimetypes filename.pict - error: unknown extension of filename.pict + error: media type unknown for filename.pict $ # now look in the extended database built into Python $ python -m mimetypes --lenient filename.pict @@ -397,7 +411,8 @@ interface: $ python -m mimetypes filename.sh filename.nc filename.xxx filename.txt type: application/x-sh encoding: None type: application/x-netcdf encoding: None - error: unknown extension of filename.xxx + error: media type unknown for filename.xxx + type: text/plain encoding: None $ # try to feed an unknown MIME type $ python -m mimetypes --extension audio/aac audio/opus audio/future audio/x-wav diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 187143d02cd7bfb..2d13053915830b0 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -100,10 +100,10 @@ To show the individual process IDs involved, here is an expanded example:: For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. -The arguments to :class:`Process` usually need to be unpickleable from within -the child process. If you tried typing the above example directly into a REPL it -could lead to an :exc:`AttributeError` in the child process trying to locate the -*f* function in the ``__main__`` module. +The arguments to :class:`Process` usually need to be picklable so they can be +passed to the child process. If you tried typing the above example directly +into a REPL it could lead to an :exc:`AttributeError` in the child process +trying to locate the *f* function in the ``__main__`` module. .. _multiprocessing-start-methods: @@ -1723,7 +1723,10 @@ inherited by child processes. Note that *lock* is a keyword only argument. Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. + attributes which can both be used to store and retrieve byte strings. + While *raw* allows interaction with a :class:`bytes` object the full size of + the array, reading *value* will terminate after a null byte, like most + programming languages handle strings. The :mod:`!multiprocessing.sharedctypes` module diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index c0dab83977e427f..3d1c8cda13be381 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -110,7 +110,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: and_(a, b) __and__(a, b) - Return the bitwise and of *a* and *b*. + Return ``a & b``. .. function:: floordiv(a, b) @@ -134,13 +134,13 @@ The mathematical and bitwise operations are the most numerous: __inv__(obj) __invert__(obj) - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. + Return ``~obj``. .. function:: lshift(a, b) __lshift__(a, b) - Return *a* shifted left by *b*. + Return ``a << b``. .. function:: mod(a, b) @@ -152,7 +152,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: mul(a, b) __mul__(a, b) - Return ``a * b``, for *a* and *b* numbers. + Return ``a * b``. .. function:: matmul(a, b) @@ -172,25 +172,25 @@ The mathematical and bitwise operations are the most numerous: .. function:: or_(a, b) __or__(a, b) - Return the bitwise or of *a* and *b*. + Return ``a | b``. .. function:: pos(obj) __pos__(obj) - Return *obj* positive (``+obj``). + Return ``+obj``. .. function:: pow(a, b) __pow__(a, b) - Return ``a ** b``, for *a* and *b* numbers. + Return ``a ** b``. .. function:: rshift(a, b) __rshift__(a, b) - Return *a* shifted right by *b*. + Return ``a >> b``. .. function:: sub(a, b) @@ -209,7 +209,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: xor(a, b) __xor__(a, b) - Return the bitwise exclusive or of *a* and *b*. + Return ``a ^ b``. Operations which work with sequences (some of them with mappings too) include: @@ -403,13 +403,18 @@ Python syntax and the functions in the :mod:`!operator` module. +-----------------------+-------------------------+---------------------------------------+ | Division | ``a // b`` | ``floordiv(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | +| Bitwise And, or | ``a & b`` | ``and_(a, b)`` | +| Intersection | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | +| Bitwise Exclusive Or, | ``a ^ b`` | ``xor(a, b)`` | +| or Symmetric | | | +| Difference | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | +| Bitwise Inversion, or | ``~ a`` | ``invert(a)`` | +| Complement | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | +| Bitwise Or, or | ``a | b`` | ``or_(a, b)`` | +| Union | | | +-----------------------+-------------------------+---------------------------------------+ | Exponentiation | ``a ** b`` | ``pow(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ diff --git a/Doc/library/os.rst b/Doc/library/os.rst index d2534b3e974f368..7b043f257ca0b57 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -219,6 +219,14 @@ process and user. :data:`os.environ`, and when one of the :meth:`~dict.pop` or :meth:`~dict.clear` methods is called. + If the :manpage:`clearenv(3)` function is available, the :meth:`~dict.clear` method + uses it and emits a single ``os._clearenv`` audit event. Otherwise, it emits + an ``os.unsetenv`` event on each deleted variable. + + .. audit-event:: os.unsetenv key os.unsetenv + + .. audit-event:: os._clearenv "" os._clearenv + .. seealso:: The :func:`os.reload_environ` function. @@ -226,6 +234,10 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.15 + The :meth:`~dict.clear` method can now emit an ``os._clearenv`` audit + event. + .. data:: environb @@ -529,7 +541,7 @@ process and user. Return a tuple (ruid, euid, suid) denoting the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -539,7 +551,7 @@ process and user. Return a tuple (rgid, egid, sgid) denoting the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -713,7 +725,7 @@ process and user. Set the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -722,7 +734,7 @@ process and user. Set the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -788,29 +800,19 @@ process and user. single: gethostbyaddr() (in module socket) Returns information identifying the current operating system. - The return value is an object with five attributes: - - * :attr:`sysname` - operating system name - * :attr:`nodename` - name of machine on network (implementation-defined) - * :attr:`release` - operating system release - * :attr:`version` - operating system version - * :attr:`machine` - hardware identifier - - For backwards compatibility, this object is also iterable, behaving - like a five-tuple containing :attr:`sysname`, :attr:`nodename`, - :attr:`release`, :attr:`version`, and :attr:`machine` - in that order. - - Some systems truncate :attr:`nodename` to 8 characters or to the - leading component; a better way to get the hostname is - :func:`socket.gethostname` or even - ``socket.gethostbyaddr(socket.gethostname())``. + The return value is a :class:`uname_result`. On macOS, iOS and Android, this returns the *kernel* name and version (i.e., ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname` can be used to get the user-facing operating system name and version on iOS and Android. + .. seealso:: + :data:`sys.platform` which has finer granularity. + + The :mod:`platform` module provides detailed checks for the + system's identity. + .. availability:: Unix. .. versionchanged:: 3.3 @@ -818,6 +820,41 @@ process and user. with named attributes. +.. class:: uname_result + + Name and information about the system returned by :func:`os.uname`. + These attributes correspond to the members described in :manpage:`uname(2)`. + + For backwards compatibility, this object is also iterable, behaving + like a five-tuple containing :attr:`~uname_result.sysname`, + :attr:`~uname_result.nodename`, :attr:`~uname_result.release`, + :attr:`~uname_result.version`, and :attr:`~uname_result.machine` + in that order. + + .. attribute:: sysname + + Operating system name. + + .. attribute:: nodename + + Name of machine on network. Some systems truncate + :attr:`~uname_result.nodename` to 8 characters or to the leading + component; a better way to get the hostname is :func:`socket.gethostname` + or even ``socket.gethostbyaddr(socket.gethostname())``. + + .. attribute:: release + + Operating system release. + + .. attribute:: version + + Operating system version. + + .. attribute:: machine + + Hardware identifier. + + .. function:: unsetenv(key, /) .. index:: single: environment variables; deleting @@ -1059,10 +1096,7 @@ as internal buffering of data. Force write of file with filedescriptor *fd* to disk. Does not force update of metadata. - .. availability:: Unix. - - .. note:: - This function is not available on MacOS. + .. availability:: Unix, not macOS, not iOS. .. function:: fpathconf(fd, name, /) @@ -1100,8 +1134,8 @@ as internal buffering of data. .. function:: fstatvfs(fd, /) Return information about the filesystem containing the file associated with - file descriptor *fd*, like :func:`statvfs`. As of Python 3.3, this is - equivalent to ``os.statvfs(fd)``. + file descriptor *fd* in a :class:`statvfs_result`, like :func:`statvfs`. + As of Python 3.3, this is equivalent to ``os.statvfs(fd)``. .. availability:: Unix. @@ -1414,7 +1448,7 @@ or `the MSDN `_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.3 @@ -1424,7 +1458,7 @@ or `the MSDN `_ on Windo Ensures that enough disk space is allocated for the file specified by *fd* starting from *offset* and continuing for *len* bytes. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1439,7 +1473,7 @@ or `the MSDN `_ on Windo :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -2549,7 +2583,8 @@ features: Windows now handles a *mode* of ``0o700``. -.. function:: makedirs(name, mode=0o777, exist_ok=False) +.. function:: makedirs(name, mode=0o777, exist_ok=False, *, \ + parent_mode=None) .. index:: single: directory; creating @@ -2567,6 +2602,12 @@ features: If *exist_ok* is ``False`` (the default), a :exc:`FileExistsError` is raised if the target directory already exists. + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories. Like *mode*, it is + combined with the process's umask value; see :ref:`the mkdir() + description `. Otherwise, intermediate directories are + created with the default mode, which is also subject to the umask. + .. note:: :func:`makedirs` will become confused if the path elements to create @@ -2593,6 +2634,11 @@ features: The *mode* argument no longer affects the file permission bits of newly created intermediate-level directories. + .. versionadded:: 3.15 + The *parent_mode* parameter. To match the behavior from Python 3.6 and + earlier (where *mode* was applied to all created directories), pass + ``parent_mode=mode``. + .. function:: mkfifo(path, mode=0o666, *, dir_fd=None) @@ -2970,6 +3016,9 @@ features: To be directly usable as a :term:`path-like object`, ``os.DirEntry`` implements the :class:`PathLike` interface. + :class:`!DirEntry` objects are :ref:`generic ` over the type of the + path (:class:`str` or :class:`bytes`). + Attributes and methods on a ``os.DirEntry`` instance are as follows: .. attribute:: name @@ -3757,48 +3806,174 @@ features: .. function:: statvfs(path) - Perform a :c:func:`!statvfs` system call on the given path. The return value is - an object whose attributes describe the filesystem on the given path, and - correspond to the members of the :c:struct:`statvfs` structure, namely: - :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, - :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, - :attr:`f_flag`, :attr:`f_namemax`, :attr:`f_fsid`. - - Two module-level constants are defined for the :attr:`f_flag` attribute's - bit-flags: if :const:`ST_RDONLY` is set, the filesystem is mounted - read-only, and if :const:`ST_NOSUID` is set, the semantics of - setuid/setgid bits are disabled or not supported. - - Additional module-level constants are defined for GNU/glibc based systems. - These are :const:`ST_NODEV` (disallow access to device special files), - :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` - (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), - :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` - (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` - (do not update access times), :const:`ST_NODIRATIME` (do not update directory access - times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). + Perform a :manpage:`statvfs(3)` system call on the given path. The return value + is a :class:`statvfs_result` whose attributes describe the filesystem + on the given path and correspond to the members of the :c:struct:`statvfs` + structure. This function can support :ref:`specifying a file descriptor `. .. availability:: Unix. - .. versionchanged:: 3.2 - The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. - .. versionchanged:: 3.4 - The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, - :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, - :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, - and :const:`ST_RELATIME` constants were added. - .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.7 - Added the :attr:`f_fsid` attribute. + +.. class:: statvfs_result + + Filesystem statistics returned by :func:`os.statvfs` and :func:`os.fstatvfs`. + See :manpage:`statvfs(3)` for more details. + + .. attribute:: f_bsize + + Block size. + + .. attribute:: f_frsize + + Fragment size. + + .. attribute:: f_blocks + + Number of :attr:`~statvfs_result.f_frsize` sized blocks the filesystem + can contain. + + .. attribute:: f_bfree + + Number of free blocks. + + .. attribute:: f_bavail + + Number of free blocks for unprivileged users. + + .. attribute:: f_files + + Number of file entries, inodes, the filesystem can contain. + + .. attribute:: f_ffree + + Number of free files entries. + + .. attribute:: f_favail + + Number of free file entries for unprivileged users. + + .. attribute:: f_flag + + Bit-mask of mount flags. The following flags are defined: + :data:`ST_RDONLY`, :data:`ST_NOSUID`, :data:`ST_NODEV`, + :data:`ST_NOEXEC`, :data:`ST_SYNCHRONOUS`, :data:`ST_MANDLOCK`, + :data:`ST_WRITE`, :data:`ST_APPEND`, :data:`ST_IMMUTABLE`, + :data:`ST_NOATIME`, :data:`ST_NODIRATIME`, and :data:`ST_RELATIME`. + + .. attribute:: f_namemax + + Filesystem max filename length. OS specific limitations such as + :ref:`Windows MAX_PATH ` and those described in Linux + :manpage:`pathname(7)` may exist. + + .. attribute:: f_fsid + + Filesystem ID. + + .. versionadded:: 3.7 + + +The following flags are used in :attr:`statvfs_result.f_flag`. + +.. data:: ST_RDONLY + + Read-only filesystem. + + .. versionadded:: 3.2 + +.. data:: ST_NOSUID + + Setuid/setgid bits are disabled or not supported. + + .. versionadded:: 3.2 + +.. data:: ST_NODEV + + Disallow access to device special files. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOEXEC + + Disallow program execution. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_SYNCHRONOUS + + Writes are synced at once. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_MANDLOCK + + Allow mandatory locks on an FS. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_WRITE + + Write on file/directory/symlink. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_APPEND + + Append-only file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_IMMUTABLE + + Immutable file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOATIME + + Do not update access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NODIRATIME + + Do not update directory access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_RELATIME + + Update atime relative to mtime/ctime. + + .. availability:: Linux. + + .. versionadded:: 3.4 .. data:: supports_dir_fd @@ -5062,13 +5237,25 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Linux >= 5.10 .. versionadded:: 3.12 +.. function:: pidfd_getfd(pidfd, targetfd, *, flags=0) + + Duplicate *targetfd* from the process referred to by the process file + descriptor *pidfd*, into the calling process. The returned file descriptor + is :ref:`non-inheritable `. + + *flags* is reserved, and currently must be ``0``. + + See the :manpage:`pidfd_getfd(2)` man page for more details. + + .. availability:: Linux >= 5.6, Android >= :func:`build-time ` API level 31 + .. versionadded:: next .. function:: plock(op, /) Lock program segments into memory. The value of *op* (defined in ````) determines which segments are locked. - .. availability:: Unix, not WASI, not iOS. + .. availability:: Unix, not WASI, not macOS, not iOS. .. function:: popen(cmd, mode='r', buffering=-1) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2867015042ee162..ab92c142c37a4f6 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1266,6 +1266,8 @@ Reading and writing files >>> p.read_text() 'Text file contents' + Return the number of characters written. + An existing file of the same name is overwritten. The optional parameters have the same meaning as in :func:`open`. @@ -1286,6 +1288,8 @@ Reading and writing files >>> p.read_bytes() b'Binary file contents' + Return the number of bytes written. + An existing file of the same name is overwritten. .. versionadded:: 3.5 @@ -1514,7 +1518,8 @@ Creating files and directories :meth:`~Path.write_bytes` methods are often used to create files. -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) +.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False, *, \ + parent_mode=None) Create a new directory at this given path. If *mode* is given, it is combined with the process's ``umask`` value to determine the file mode @@ -1525,6 +1530,12 @@ Creating files and directories as needed; they are created with the default permissions without taking *mode* into account (mimicking the POSIX ``mkdir -p`` command). + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories when *parents* is true. + Like *mode*, it is combined with the process's ``umask`` value. + Otherwise, intermediate directories are created with the default + permissions (also subject to the umask). + If *parents* is false (the default), a missing parent raises :exc:`FileNotFoundError`. @@ -1538,6 +1549,9 @@ Creating files and directories .. versionchanged:: 3.5 The *exist_ok* parameter was added. + .. versionadded:: 3.15 + The *parent_mode* parameter. + .. method:: Path.symlink_to(target, target_is_directory=False) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index f8975c2f4281d45..8eadc2cf2b1ef0d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -56,7 +56,7 @@ The :mod:`!pickle` module differs from :mod:`marshal` in several significant way * :mod:`marshal` cannot be used to serialize user-defined classes and their instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support @@ -693,7 +693,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index e753ad3b08b81a1..769ca60af1837ea 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -79,7 +79,7 @@ Command-line options A pickle file to read, or ``-`` to indicate reading from standard input. -.. versionadded:: next +.. versionadded:: 3.15 Output is in color by default and can be :ref:`controlled using environment variables `. diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index f236eba84576575..a0cfb440a36ffa9 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -68,6 +68,11 @@ will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. +.. warning:: + + The :mod:`!pydoc` HTTP server is intended for local use during + development and is not suitable for production use. + :program:`python -m pydoc -n ` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 2e6938b5cf68602..c88411ce0b7b91f 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -63,12 +63,26 @@ The :mod:`!xml.parsers.expat` module contains two functions: .. function:: ParserCreate(encoding=None, namespace_separator=None) - Creates and returns a new :class:`xmlparser` object. *encoding*, if specified, - must be a string naming the encoding used by the XML data. Expat doesn't - support as many encodings as Python does, and its repertoire of encodings can't - be extended; it supports UTF-8, UTF-16, ISO-8859-1 (Latin1), and ASCII. If - *encoding* [1]_ is given it will override the implicit or explicit encoding of the - document. + Creates and returns a new :class:`xmlparser` object. + *encoding* [1]_, if specified, must be a string naming the encoding + used by the XML data. + If it is given it will override the implicit or explicit encoding + of the document. + + .. impl-detail:: + + Expat natively understands and processes UTF-8, UTF-16, UTF-16BE, + UTF-16LE, ISO-8859-1, and US-ASCII. + For other encodings (including aliases like Latin1 and ASCII) it + falls back to Python. + It supports most of 8-bit encodings and many multi-byte encodings + like Shift_JIS, although only BMP characters (``U+0000-U+FFFF``) + are supported with non-native encodings (this restriction is also + applied to aliases like UTF8). + These restrictions only apply if *encoding* is not given. + + .. versionchanged:: next + Added support for multi-byte encodings. .. _xmlparser-non-root: @@ -113,7 +127,6 @@ The :mod:`!xml.parsers.expat` module contains two functions: XML document. Call ``ParserCreate`` for each document to provide unique parser instances. - .. seealso:: `The Expat XML Parser `_ @@ -1083,9 +1096,11 @@ The ``errors`` module has the following attributes: .. rubric:: Footnotes -.. [1] The encoding string included in XML output should conform to the - appropriate standards. For example, "UTF-8" is valid, but "UTF8" is - not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl +.. [1] The encoding string included in XML output should conform to + the appropriate standards. For example, "UTF-8" is valid, but + "UTF8" is not valid in an XML document's declaration, even though + Python accepts it as an encoding name. + See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 5ac72ef7604d50c..79da2a3db719e78 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -76,6 +76,8 @@ The :mod:`!queue` module defines the following classes and exceptions: Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. Simple queues lack advanced functionality such as task tracking. + Simple queues are :ref:`generic ` over the type of their items. + .. versionadded:: 3.7 @@ -171,7 +173,7 @@ provide the public methods described below. .. method:: Queue.get_nowait() - Equivalent to ``get(False)``. + Equivalent to ``get(block=False)``. Two methods are offered to support tracking whether enqueued tasks have been fully processed by daemon consumer threads. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index a46fd42458158cf..7c8c589b3f5dfcb 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -279,25 +279,47 @@ The special characters are: ``[]()[{}]`` will match a right bracket, as well as left bracket, braces, and parentheses. - .. .. index:: single: --; in regular expressions - .. .. index:: single: &&; in regular expressions - .. .. index:: single: ~~; in regular expressions - .. .. index:: single: ||; in regular expressions - - * Support of nested sets and set operations as in `Unicode Technical - Standard #18`_ might be added in the future. This would change the - syntax, so to facilitate this change a :exc:`FutureWarning` will be raised - in ambiguous cases for the time being. - That includes sets starting with a literal ``'['`` or containing literal - character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To - avoid a warning escape them with a backslash. + .. index:: + single: --; in regular expressions + single: &&; in regular expressions + single: ||; in regular expressions + + * A character set may contain a nested set written in square brackets, and + two sets may be combined with a set operator, as in `Unicode Technical + Standard #18`_: + + * ``[A--B]`` (*difference*) matches a character that is in *A* but not + in *B*; for example ``[a-z--[aeiou]]`` matches an ASCII lowercase + consonant. + * ``[A&&B]`` (*intersection*) matches a character that is in both *A* + and *B*; for example ``[\w&&[a-z]]`` matches an ASCII lowercase letter. + * ``[A||B]`` (*union*) matches a character that is in *A* or in *B*; this + is the same as listing the members of both sets in a single set, but + allows combining nested sets. + + Operators have no precedence and are applied from left to right. To + group, write a nested set as the operand after an operator, as in + ``[a-z--[aeiou]]``. A leading ``'^'`` complements the whole result. + A ``'['`` begins a nested set only immediately after a set operator; + anywhere else -- including at the start of a character set -- it is an + ordinary character, so existing patterns keep their meaning. Escape it + as ``'\['`` to include a literal ``'['`` right after an operator. .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + .. note:: + + Symmetric difference (``A~~B``) is not yet supported; a literal ``'~~'`` + in a character set still raises a :exc:`FutureWarning`. + .. versionchanged:: 3.7 :exc:`FutureWarning` is raised if a character set contains constructs that will change semantically in the future. + .. versionchanged:: next + Added support for nested sets and the set operators ``--``, ``&&`` + and ``||``. + .. index:: single: | (vertical bar); in regular expressions ``|`` @@ -1257,6 +1279,9 @@ Regular expression objects Compiled regular expression object returned by :func:`re.compile`. + Patterns are :ref:`generic ` over the type of string they handle + (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. See :ref:`types-genericalias`. @@ -1419,6 +1444,9 @@ when there is no match, you can test whether there was a match with a simple Match object returned by successful ``match``\ es and ``search``\ es. + Matches are :ref:`generic ` over the type of string which was + matched (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. See :ref:`types-genericalias`. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 536b5980f86a817..d764a98c8419ed3 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -106,7 +106,7 @@ The :mod:`!runpy` module provides two functions: Execute the code at the named filesystem location and return the resulting module's globals dictionary. As with a script name supplied to the CPython - command line, *file_path* may refer to a Python source file, a + command line, *path_name* may refer to a Python source file, a compiled bytecode file or a valid :data:`sys.path` entry containing a :mod:`__main__` module (e.g. a zipfile containing a top-level :file:`__main__.py` file). @@ -134,12 +134,12 @@ The :mod:`!runpy` module provides two functions: ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``''`` otherwise. - If *file_path* directly references a script file (whether as source + If *path_name* directly references a script file (whether as source or as precompiled byte code), then ``__file__`` will be set to - *file_path*, and ``__spec__``, ``__loader__`` and + *path_name*, and ``__spec__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If *file_path* is a reference to a valid :data:`sys.path` entry, then + If *path_name* is a reference to a valid :data:`sys.path` entry, then ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__loader__`` and ``__package__`` will be @@ -147,7 +147,7 @@ The :mod:`!runpy` module provides two functions: A number of alterations are also made to the :mod:`sys` module. Firstly, :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated - with the value of *file_path* and ``sys.modules[__name__]`` is updated + with the value of *path_name* and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function returns. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 70541c5f3cb3676..037e27f031d0c82 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -117,9 +117,11 @@ Scheduler Objects function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). + If *blocking* is false, immediately executes all events in the queue which have + a time value less than or equal to the current *timefunc* value (if any) and + returns the difference between the current *timefunc* value and the time value + of the next scheduled event in the scheduler's event queue. If the queue is + empty, returns ``None``. Either *action* or *delayfunc* can raise an exception. In either case, the scheduler will maintain a consistent state and propagate the exception. If an diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 09563af14d018a4..6400005871746a5 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,9 +54,11 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. @@ -94,18 +96,22 @@ The module defines the following: When CPython is built, this function may be disabled using :option:`--disable-epoll`. + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -113,12 +119,16 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + .. function:: select(rlist, wlist, xlist, timeout=None) @@ -190,7 +200,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -285,52 +295,52 @@ object. .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -362,12 +372,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -396,7 +406,7 @@ Edge and Level Trigger Polling (epoll) Objects .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -421,24 +431,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -489,7 +499,7 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -533,7 +543,7 @@ Kqueue Objects .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -553,66 +563,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permits control() to return the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disables event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -620,19 +630,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -640,22 +650,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -663,19 +673,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 2ab12f2f6f9169e..2dfb0246d5d90c0 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -44,12 +44,15 @@ The :mod:`!shlex` module defines the following functions: .. versionadded:: 3.8 -.. function:: quote(s) +.. function:: quote(s, *, force=False) Return a shell-escaped version of the string *s*. The returned value is a string that can safely be used as one token in a shell command line, for cases where you cannot use a list. + If *force* is :const:`True`, then *s* is unconditionally quoted, + even if it is already safe for a shell without being quoted. + .. _shlex-quote-warning: .. warning:: @@ -91,8 +94,23 @@ The :mod:`!shlex` module defines the following functions: >>> command ['ls', '-l', 'somefile; rm -rf ~'] + The *force* keyword can be used to produce consistent behavior when + escaping multiple strings: + + >>> from shlex import quote + >>> filenames = ['my first file', 'file2', 'file 3'] + >>> filenames_some_escaped = [quote(f) for f in filenames] + >>> filenames_some_escaped + ["'my first file'", 'file2', "'file 3'"] + >>> filenames_all_escaped = [quote(f, force=True) for f in filenames] + >>> filenames_all_escaped + ["'my first file'", "'file2'", "'file 3'"] + .. versionadded:: 3.3 + .. versionchanged:: next + The *force* keyword was added. + The :mod:`!shlex` module defines the following class: diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d289ba58c240658..6febc7a187a15f8 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -385,10 +385,14 @@ Directory and files operations If *dst* already exists but is not a directory, it may be overwritten depending on :func:`os.rename` semantics. - If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to the destination using *copy_function* - and then removed. In case of symlinks, a new symlink pointing to the target - of *src* will be created as the destination and *src* will be removed. + :func:`os.rename` is preferably used internally when *src* and the destination are on + the same filesystem. In case :func:`os.rename` fails due to :exc:`OSError` + (e.g. the user has write permission to the destination file but not to its parent + directory), this method falls back to using *copy_function*, in which case + *src* is copied to the destination using *copy_function* and then removed. + + In case of symlinks, a new symlink pointing to the target of *src* will be + created in or as the destination, and *src* will be removed. If *copy_function* is given, it must be a callable that takes two arguments, *src* and the destination, and will be used to copy *src* to the destination @@ -696,7 +700,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Register an archiver for the format *name*. - *function* is the callable that will be used to unpack archives. The callable + *function* is the callable that will be used to create archives. The callable will receive the *base_name* of the file to create, followed by the *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. Further arguments are passed as keyword arguments: *owner*, *group*, @@ -749,8 +753,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. Since Python 3.14, the defaults for both built-in formats (zip and tar files) will prevent the most dangerous of such security issues, diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 3703d2fa60056f0..11a5484c2b13362 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -96,7 +96,7 @@ or tab) are executed with :func:`exec`. The :file:`.pth` files are now decoded by UTF-8 at first and then by the :term:`locale encoding` if it fails. -.. versionchanged:: next +.. versionchanged:: 3.15 :file:`.pth` file lines starting with ``import`` are deprecated. During the deprecation period, such lines are still executed (except in the case @@ -110,7 +110,7 @@ or tab) are executed with :func:`exec`. file. Each error is reported and the remaining lines continue to be processed. -.. deprecated-removed:: next 3.20 +.. deprecated-removed:: 3.15 3.20 Decoding :file:`{name}.pth` files in any encoding other than ``utf-8-sig`` is deprecated in Python 3.15, and support for decoding from the locale @@ -126,7 +126,7 @@ or tab) are executed with :func:`exec`. Startup entry points (:file:`.start` files) ------------------------------------------- -.. versionadded:: next +.. versionadded:: 3.15 A startup entry point file is a file whose name has the form :file:`{name}.start` and exists in one of the site-packages directories @@ -356,7 +356,79 @@ Module contents This function used to be called unconditionally. -.. function:: addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False) +.. function:: makepath(*paths) + + Join *paths* with :func:`os.path.join`, attempt to make the result + absolute with :func:`os.path.abspath`, and return a 2-tuple containing + the absolute path and its case-normalized form as produced by + :func:`os.path.normcase`. If :func:`os.path.abspath` raises + :exc:`OSError`, the joined path is used unchanged for the + case-normalization step. + + The second element of the returned tuple is the form used throughout the + :mod:`!site` module to compare paths on case-insensitive file systems, and + is what populates the ``known_paths`` sets that prevent duplicate + :data:`sys.path` entries in various APIs within this module. + + +.. class:: StartupState(known_paths=None) + + Instances of this class accumulate interpreter startup configuration data + from one or more site directories. They are the preferred interface for + batching the processing of :file:`.pth` and :file:`.start` files across + multiple site directories, so that every :data:`sys.path` extension is + visible before any startup code runs. + + The optional *known_paths* argument is a set of case-normalized paths + (which can be produced by :func:`makepath`) used to prevent duplicate + :data:`sys.path` entries. When ``None`` (the default), the set is built + from the current :data:`sys.path`. :func:`main` implicitly uses an + instance of this class. + + Typical use: + + .. code-block:: python + + state = site.StartupState() + for sitedir in site_dirs: + state.addsitedir(sitedir) + state.process() + + .. versionadded:: 3.15 + + .. method:: addsitedir(sitedir) + + Read the :file:`.pth` and :file:`.start` files in *sitedir* and + record their :data:`sys.path` extensions, deprecated :file:`.pth` + ``import`` lines, and :file:`.start` entry points on this state. + The recorded data is not applied until :meth:`process` is called. + + .. method:: addusersitepackages() + + Add the per-user site-packages directory, if enabled and if it exists. + The directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: addsitepackages(prefixes=None) + + Add global site-packages directories, computed from *prefixes* or from + the global :data:`PREFIXES` when *prefixes* is ``None``. Each + directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: process() + + Apply the accumulated state by first adding the path extensions to + :data:`sys.path`, then executing the :file:`.start` file entry points + and :file:`.pth` file ``import`` lines (:ref:`deprecated + `). + + This method is not idempotent and must not be called more than once + on the same instance. Doing so will apply the accumulated state + more than once, re-running entry points and ``import`` lines. + + +.. function:: addsitedir(sitedir, known_paths=None) Add a directory to sys.path and parse the :file:`.pth` and :file:`.start` files found in that directory. Typically used in :mod:`sitecustomize` or @@ -366,17 +438,15 @@ Module contents used to prevent duplicate :data:`sys.path` entries. When ``None`` (the default), the set is built from the current :data:`sys.path`. - While :file:`.pth` and :file:`.start` files are always parsed, set - *defer_processing_start_files* to ``True`` to prevent processing the - startup data found in those files, so that you can process them explicitly - (this is typically used by the :func:`main` function). + For batched processing across multiple site directories, build a + :class:`StartupState` explicitly and call :meth:`StartupState.addsitedir` + on it; that defers :file:`.pth` and :file:`.start` processing until a + single :meth:`StartupState.process` call, ensuring every :data:`sys.path` + extension is visible before any startup code runs. - .. versionchanged:: next + .. versionchanged:: 3.15 Also processes :file:`.start` files. See :ref:`site-start-files`. - All :file:`.pth` and :file:`.start` files are now read and - accumulated before any path extensions, ``import`` line execution, - or entry point invocations take place. .. function:: getsitepackages() @@ -447,4 +517,3 @@ value greater than 2 if there is an error. * :pep:`370` -- Per user site-packages directory * :pep:`829` -- Startup entry points and the deprecation of import lines in ``.pth`` files * :ref:`sys-path-init` -- The initialization of :data:`sys.path`. - diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 96bc9e7a0d61e3d..836aa91bb0885b1 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -842,65 +842,8 @@ Creating sockets The following functions all create :ref:`socket objects `. -.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) - - Create a new socket using the given address family, socket type and protocol - number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, - or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the - default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other - ``SOCK_`` constants. The protocol number is usually zero and may be omitted - or in the case where the address family is :const:`AF_CAN` the protocol - should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or - :const:`CAN_J1939`. - - If *fileno* is specified, the values for *family*, *type*, and *proto* are - auto-detected from the specified file descriptor. Auto-detection can be - overruled by calling the function with explicit *family*, *type*, or *proto* - arguments. This only affects how Python represents e.g. the return value - of :meth:`socket.getpeername` but not the actual OS resource. Unlike - :func:`socket.fromfd`, *fileno* will return the same socket and not a - duplicate. This may help close a detached socket using - :meth:`socket.close`. - - The newly created socket is :ref:`non-inheritable `. - - .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - - .. versionchanged:: 3.3 - The AF_CAN family was added. - The AF_RDS family was added. - - .. versionchanged:: 3.4 - The CAN_BCM protocol was added. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - .. versionchanged:: 3.7 - The CAN_ISOTP protocol was added. - - .. versionchanged:: 3.7 - When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` - bit flags are applied to *type* they are cleared, and - :attr:`socket.type` will not reflect them. They are still passed - to the underlying system ``socket()`` call. Therefore, - - :: - - sock = socket.socket( - socket.AF_INET, - socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - - will still create a non-blocking socket on OSes that support - ``SOCK_NONBLOCK``, but ``sock.type`` will be set to - ``socket.SOCK_STREAM``. - - .. versionchanged:: 3.9 - The CAN_J1939 protocol was added. - - .. versionchanged:: 3.10 - The IPPROTO_MPTCP protocol was added. +The :class:`socket ` class constructor creates a new socket +directly; see :ref:`socket-objects` for its parameters and full description. .. function:: socketpair([family[, type[, proto]]]) @@ -1025,12 +968,6 @@ The following functions all create :ref:`socket objects `. .. versionadded:: 3.3 -.. data:: SocketType - - This is a Python type object that represents the socket object type. It is the - same as ``type(socket(...))``. - - Other functions ''''''''''''''' @@ -1538,642 +1475,709 @@ The :mod:`!socket` module also offers various network-related services: Socket Objects -------------- -Socket objects have the following methods. Except for -:meth:`~socket.makefile`, these correspond to Unix system calls applicable -to sockets. +.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) -.. versionchanged:: 3.2 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`~socket.close`. + Create a new socket using the given address family, socket type and protocol + number. The address family should be :const:`AF_INET` (the default), + :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, + or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the + default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other + ``SOCK_`` constants. The protocol number is usually zero and may be omitted + or in the case where the address family is :const:`AF_CAN` the protocol + should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or + :const:`CAN_J1939`. + If *fileno* is specified, the values for *family*, *type*, and *proto* are + auto-detected from the specified file descriptor. Auto-detection can be + overruled by calling the function with explicit *family*, *type*, or *proto* + arguments. This only affects how Python represents e.g. the return value + of :meth:`socket.getpeername` but not the actual OS resource. Unlike + :func:`socket.fromfd`, *fileno* will return the same socket and not a + duplicate. This may help close a detached socket using + :meth:`socket.close`. -.. method:: socket.accept() + The newly created socket is :ref:`non-inheritable `. - Accept a connection. The socket must be bound to an address and listening for - connections. The return value is a pair ``(conn, address)`` where *conn* is a - *new* socket object usable to send and receive data on the connection, and - *address* is the address bound to the socket on the other end of the connection. + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.3 + The AF_CAN family was added. + The AF_RDS family was added. .. versionchanged:: 3.4 - The socket is now non-inheritable. + The CAN_BCM protocol was added. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.4 + The returned socket is now non-inheritable. + .. versionchanged:: 3.7 + The CAN_ISOTP protocol was added. -.. method:: socket.bind(address) + .. versionchanged:: 3.7 + When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` + bit flags are applied to *type* they are cleared, and + :attr:`socket.type` will not reflect them. They are still passed + to the underlying system ``socket()`` call. Therefore, - Bind the socket to *address*. The socket must not already be bound. The format - of *address* depends on the address family --- see :ref:`socket-addresses`. + :: - .. audit-event:: socket.bind self,address socket.socket.bind + sock = socket.socket( + socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - .. availability:: not WASI. + will still create a non-blocking socket on OSes that support + ``SOCK_NONBLOCK``, but ``sock.type`` will be set to + ``socket.SOCK_STREAM``. + .. versionchanged:: 3.9 + The CAN_J1939 protocol was added. -.. method:: socket.close() + .. versionchanged:: 3.10 + The IPPROTO_MPTCP protocol was added. - Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile` - are closed. Once that happens, all future operations on the socket - object will fail. The remote end will receive no more data (after - queued data is flushed). + Socket objects have the following methods. Except for + :meth:`~socket.makefile`, these correspond to Unix system calls applicable + to sockets. - Sockets are automatically closed when they are garbage-collected, but - it is recommended to :meth:`close` them explicitly, or to use a - :keyword:`with` statement around them. + .. versionchanged:: 3.2 + Support for the :term:`context manager` protocol was added. Exiting the + context manager is equivalent to calling :meth:`~socket.close`. - .. versionchanged:: 3.6 - :exc:`OSError` is now raised if an error occurs when the underlying - :c:func:`close` call is made. - .. note:: + .. method:: accept() - :meth:`close` releases the resource associated with a connection but - does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown` - before :meth:`close`. + Accept a connection. The socket must be bound to an address and listening for + connections. The return value is a pair ``(conn, address)`` where *conn* is a + *new* socket object usable to send and receive data on the connection, and + *address* is the address bound to the socket on the other end of the connection. + The newly created socket is :ref:`non-inheritable `. -.. method:: socket.connect(address) + .. versionchanged:: 3.4 + The socket is now non-inheritable. - Connect to a remote socket at *address*. The format of *address* depends on the - address family --- see :ref:`socket-addresses`. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - If the connection is interrupted by a signal, the method waits until the - connection completes, or raises a :exc:`TimeoutError` on timeout, if the - signal handler doesn't raise an exception and the socket is blocking or has - a timeout. For non-blocking sockets, the method raises an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal (or the exception raised by the signal handler). - .. audit-event:: socket.connect self,address socket.socket.connect + .. method:: bind(address) - .. versionchanged:: 3.5 - The method now waits until the connection completes instead of raising an + Bind the socket to *address*. The socket must not already be bound. The format + of *address* depends on the address family --- see :ref:`socket-addresses`. + + .. audit-event:: socket.bind self,address socket.socket.bind + + .. availability:: not WASI. + + + .. method:: close() + + Mark the socket closed. The underlying system resource (e.g. a file + descriptor) is also closed when all file objects from :meth:`makefile` + are closed. Once that happens, all future operations on the socket + object will fail. The remote end will receive no more data (after + queued data is flushed). + + Sockets are automatically closed when they are garbage-collected, but + it is recommended to :meth:`close` them explicitly, or to use a + :keyword:`with` statement around them. + + .. versionchanged:: 3.6 + :exc:`OSError` is now raised if an error occurs when the underlying + :c:func:`!close` call is made. + + .. note:: + + :meth:`close` releases the resource associated with a connection but + does not necessarily close the connection immediately. If you want + to close the connection in a timely fashion, call :meth:`shutdown` + before :meth:`close`. + + + .. method:: connect(address) + + Connect to a remote socket at *address*. The format of *address* depends on the + address family --- see :ref:`socket-addresses`. + + If the connection is interrupted by a signal, the method waits until the + connection completes, or raises a :exc:`TimeoutError` on timeout, if the + signal handler doesn't raise an exception and the socket is blocking or has + a timeout. For non-blocking sockets, the method raises an :exc:`InterruptedError` exception if the connection is interrupted by a - signal, the signal handler doesn't raise an exception and the socket is - blocking or has a timeout (see the :pep:`475` for the rationale). + signal (or the exception raised by the signal handler). - .. availability:: not WASI. + .. audit-event:: socket.connect self,address socket.socket.connect + .. versionchanged:: 3.5 + The method now waits until the connection completes instead of raising an + :exc:`InterruptedError` exception if the connection is interrupted by a + signal, the signal handler doesn't raise an exception and the socket is + blocking or has a timeout (see the :pep:`475` for the rationale). -.. method:: socket.connect_ex(address) + .. availability:: not WASI. - Like ``connect(address)``, but return an error indicator instead of raising an - exception for errors returned by the C-level :c:func:`connect` call (other - problems, such as "host not found," can still raise exceptions). The error - indicator is ``0`` if the operation succeeded, otherwise the value of the - :c:data:`errno` variable. This is useful to support, for example, asynchronous - connects. - .. audit-event:: socket.connect self,address socket.socket.connect_ex + .. method:: connect_ex(address) - .. availability:: not WASI. + Like ``connect(address)``, but return an error indicator instead of raising an + exception for errors returned by the C-level :c:func:`!connect` call (other + problems, such as "host not found," can still raise exceptions). The error + indicator is ``0`` if the operation succeeded, otherwise the value of the + :c:data:`errno` variable. This is useful to support, for example, asynchronous + connects. -.. method:: socket.detach() + .. audit-event:: socket.connect self,address socket.socket.connect_ex - Put the socket object into closed state without actually closing the - underlying file descriptor. The file descriptor is returned, and can - be reused for other purposes. + .. availability:: not WASI. - .. versionadded:: 3.2 + .. method:: detach() + Put the socket object into closed state without actually closing the + underlying file descriptor. The file descriptor is returned, and can + be reused for other purposes. -.. method:: socket.dup() + .. versionadded:: 3.2 - Duplicate the socket. - The newly created socket is :ref:`non-inheritable `. + .. method:: dup() - .. versionchanged:: 3.4 - The socket is now non-inheritable. + Duplicate the socket. - .. availability:: not WASI. + The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.4 + The socket is now non-inheritable. -.. method:: socket.fileno() + .. availability:: not WASI. - Return the socket's file descriptor (a small integer), or -1 on failure. This - is useful with :func:`select.select`. - Under Windows the small integer returned by this method cannot be used where a - file descriptor can be used (such as :func:`os.fdopen`). Unix does not have - this limitation. + .. method:: fileno() -.. method:: socket.get_inheritable() + Return the socket's file descriptor (a small integer), or -1 on failure. This + is useful with :func:`select.select`. - Get the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle: ``True`` if the socket can be inherited in - child processes, ``False`` if it cannot. + Under Windows the small integer returned by this method cannot be used where a + file descriptor can be used (such as :func:`os.fdopen`). Unix does not have + this limitation. - .. versionadded:: 3.4 + .. method:: get_inheritable() + Get the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle: ``True`` if the socket can be inherited in + child processes, ``False`` if it cannot. -.. method:: socket.getpeername() + .. versionadded:: 3.4 - Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. The format - of the address returned depends on the address family --- see :ref:`socket-addresses`. - On some systems this function is not supported. + .. method:: getpeername() -.. method:: socket.getsockname() + Return the remote address to which the socket is connected. This is useful to + find out the port number of a remote IPv4/v6 socket, for instance. The format + of the address returned depends on the address family --- see :ref:`socket-addresses`. + On some systems this function is not supported. - Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. The format of the address returned depends on - the address family --- see :ref:`socket-addresses`. + .. method:: getsockname() -.. method:: socket.getsockopt(level, optname[, buflen]) + Return the socket's own address. This is useful to find out the port number of + an IPv4/v6 socket, for instance. The format of the address returned depends on + the address family --- see :ref:`socket-addresses`. - Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) - are defined in this module. If *buflen* is absent, an integer option is assumed - and its integer value is returned by the function. If *buflen* is present, it - specifies the maximum length of the buffer used to receive the option in, and - this buffer is returned as a bytes object. It is up to the caller to decode the - contents of the buffer (see the optional built-in module :mod:`struct` for a way - to decode C structures encoded as byte strings). - .. availability:: not WASI. + .. method:: getsockopt(level, optname[, buflen]) + Return the value of the given socket option (see the Unix man page + :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) + are defined in this module. If *buflen* is absent, an integer option is assumed + and its integer value is returned by the function. If *buflen* is present, it + specifies the maximum length of the buffer used to receive the option in, and + this buffer is returned as a bytes object. It is up to the caller to decode the + contents of the buffer (see the optional built-in module :mod:`struct` for a way + to decode C structures encoded as byte strings). -.. method:: socket.getblocking() + .. availability:: not WASI. - Return ``True`` if socket is in blocking mode, ``False`` if in - non-blocking. - This is equivalent to checking ``socket.gettimeout() != 0``. + .. method:: getblocking() - .. versionadded:: 3.7 + Return ``True`` if socket is in blocking mode, ``False`` if in + non-blocking. + This is equivalent to checking ``socket.gettimeout() != 0``. -.. method:: socket.gettimeout() + .. versionadded:: 3.7 - Return the timeout in seconds (float) associated with socket operations, - or ``None`` if no timeout is set. This reflects the last call to - :meth:`setblocking` or :meth:`settimeout`. + .. method:: gettimeout() -.. method:: socket.ioctl(control, option) + Return the timeout in seconds (float) associated with socket operations, + or ``None`` if no timeout is set. This reflects the last call to + :meth:`setblocking` or :meth:`settimeout`. - The :meth:`ioctl` method is a limited interface to the WSAIoctl system - interface. Please refer to the `Win32 documentation - `_ for more - information. - On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` - functions may be used; they accept a socket object as their first argument. + .. method:: ioctl(control, option) - Currently only the following control codes are supported: - ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + The :meth:`ioctl` method is a limited interface to the WSAIoctl system + interface. Please refer to the `Win32 documentation + `_ for more + information. - .. availability:: Windows + On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` + functions may be used; they accept a socket object as their first argument. - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. + Currently only the following control codes are supported: + ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + .. availability:: Windows -.. method:: socket.listen([backlog]) + .. versionchanged:: 3.6 + ``SIO_LOOPBACK_FAST_PATH`` was added. - Enable a server to accept connections. If *backlog* is specified, it must - be at least 0 (if it is lower, it is set to 0); it specifies the number of - unaccepted connections that the system will allow before refusing new - connections. If not specified, a default reasonable value is chosen. - .. availability:: not WASI. + .. method:: listen([backlog]) - .. versionchanged:: 3.5 - The *backlog* parameter is now optional. + Enable a server to accept connections. If *backlog* is specified, it must + be at least 0 (if it is lower, it is set to 0); it specifies the number of + unaccepted connections that the system will allow before refusing new + connections. If not specified, a default reasonable value is chosen. + .. availability:: not WASI. -.. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \ - errors=None, newline=None) + .. versionchanged:: 3.5 + The *backlog* parameter is now optional. - .. index:: single: I/O control; buffering - Return a :term:`file object` associated with the socket. The exact returned - type depends on the arguments given to :meth:`makefile`. These arguments are - interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or - a combination of those. + .. method:: makefile(mode='r', buffering=None, *, encoding=None, \ + errors=None, newline=None) - The socket must be in blocking mode; it can have a timeout, but the file - object's internal buffer may end up in an inconsistent state if a timeout - occurs. + .. index:: single: I/O control; buffering - Closing the file object returned by :meth:`makefile` won't close the - original socket unless all other file objects have been closed and - :meth:`socket.close` has been called on the socket object. + Return a :term:`file object` associated with the socket. The exact returned + type depends on the arguments given to :meth:`makefile`. These arguments are + interpreted the same way as by the built-in :func:`open` function, except + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. - .. note:: + The socket must be in blocking mode; it can have a timeout, but the file + object's internal buffer may end up in an inconsistent state if a timeout + occurs. - On Windows, the file-like object created by :meth:`makefile` cannot be - used where a file object with a file descriptor is expected, such as the - stream arguments of :meth:`subprocess.Popen`. + Closing the file object returned by :meth:`makefile` won't close the + original socket unless all other file objects have been closed and + :meth:`socket.close` has been called on the socket object. + .. note:: -.. method:: socket.recv(bufsize[, flags]) + On Windows, the file-like object created by :meth:`makefile` cannot be + used where a file object with a file descriptor is expected, such as the + stream arguments of :meth:`subprocess.Popen`. - Receive data from the socket. The return value is a bytes object representing the - data received. The maximum amount of data to be received at once is specified - by *bufsize*. A returned empty bytes object indicates that the client has disconnected. - See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument - *flags*; it defaults to zero. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recv(bufsize[, flags]) + Receive data from the socket. The return value is a bytes object representing the + data received. The maximum amount of data to be received at once is specified + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. -.. method:: socket.recvfrom(bufsize[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket. The return value is a pair ``(bytes, address)`` - where *bytes* is a bytes object representing the data received and *address* is the - address of the socket sending the data. See the Unix manual page - :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. The format of *address* depends on the address family --- see - :ref:`socket-addresses`. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recvfrom(bufsize[, flags]) - .. versionchanged:: 3.7 - For multicast IPv6 address, first item of *address* does not contain - ``%scope_id`` part anymore. In order to get full IPv6 address use - :func:`getnameinfo`. - -.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) - - Receive normal data (up to *bufsize* bytes) and ancillary data from - the socket. The *ancbufsize* argument sets the size in bytes of - the internal buffer used to receive the ancillary data; it defaults - to 0, meaning that no ancillary data will be received. Appropriate - buffer sizes for ancillary data can be calculated using - :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit - into the buffer might be truncated or discarded. The *flags* - argument defaults to 0 and has the same meaning as for - :meth:`recv`. - - The return value is a 4-tuple: ``(data, ancdata, msg_flags, - address)``. The *data* item is a :class:`bytes` object holding the - non-ancillary data received. The *ancdata* item is a list of zero - or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing - the ancillary data (control messages) received: *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - :class:`bytes` object holding the associated data. The *msg_flags* - item is the bitwise OR of various flags indicating conditions on - the received message; see your system documentation for details. - If the receiving socket is unconnected, *address* is the address of - the sending socket, if available; otherwise, its value is - unspecified. - - On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to - pass file descriptors between processes over an :const:`AF_UNIX` - socket. When this facility is used (it is often restricted to - :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its - ancillary data, items of the form ``(socket.SOL_SOCKET, - socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object - representing the new file descriptors as a binary array of the - native C :c:expr:`int` type. If :meth:`recvmsg` raises an - exception after the system call returns, it will first attempt to - close any file descriptors received via this mechanism. - - Some systems do not indicate the truncated length of ancillary data - items which have been only partially received. If an item appears - to extend beyond the end of the buffer, :meth:`recvmsg` will issue - a :exc:`RuntimeWarning`, and will return the part of it which is - inside the buffer provided it has not been truncated before the - start of its associated data. - - On systems which support the :const:`SCM_RIGHTS` mechanism, the - following function will receive up to *maxfds* file descriptors, - returning the message data and a list containing the descriptors - (while ignoring unexpected conditions such as unrelated control - messages being received). See also :meth:`sendmsg`. :: - - import socket, array - - def recv_fds(sock, msglen, maxfds): - fds = array.array("i") # Array of ints - msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: - # Append data, ignoring any truncated integers at the end. - fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - return msg, list(fds) - - .. availability:: Unix. + Receive data from the socket. The return value is a pair ``(bytes, address)`` + where *bytes* is a bytes object representing the data received and *address* is the + address of the socket sending the data. See the Unix manual page + :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults + to zero. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. - Most Unix platforms. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. versionchanged:: 3.7 + For multicast IPv6 address, first item of *address* does not contain + ``%scope_id`` part anymore. In order to get full IPv6 address use + :func:`getnameinfo`. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]]) - - Receive normal data and ancillary data from the socket, behaving as - :meth:`recvmsg` would, but scatter the non-ancillary data into a - series of buffers instead of returning a new bytes object. The - *buffers* argument must be an iterable of objects that export - writable buffers (e.g. :class:`bytearray` objects); these will be - filled with successive chunks of the non-ancillary data until it - has all been written or there are no more buffers. The operating - system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) - on the number of buffers that can be used. The *ancbufsize* and - *flags* arguments have the same meaning as for :meth:`recvmsg`. - - The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, - address)``, where *nbytes* is the total number of bytes of - non-ancillary data written into the buffers, and *ancdata*, - *msg_flags* and *address* are the same as for :meth:`recvmsg`. - - Example:: - - >>> import socket - >>> s1, s2 = socket.socketpair() - >>> b1 = bytearray(b'----') - >>> b2 = bytearray(b'0123456789') - >>> b3 = bytearray(b'--------------') - >>> s1.send(b'Mary had a little lamb') - 22 - >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) - (22, [], 0, None) - >>> [b1, b2, b3] - [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] - - .. availability:: Unix. + .. method:: recvmsg(bufsize[, ancbufsize[, flags]]) + + Receive normal data (up to *bufsize* bytes) and ancillary data from + the socket. The *ancbufsize* argument sets the size in bytes of + the internal buffer used to receive the ancillary data; it defaults + to 0, meaning that no ancillary data will be received. Appropriate + buffer sizes for ancillary data can be calculated using + :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit + into the buffer might be truncated or discarded. The *flags* + argument defaults to 0 and has the same meaning as for + :meth:`recv`. + + The return value is a 4-tuple: ``(data, ancdata, msg_flags, + address)``. The *data* item is a :class:`bytes` object holding the + non-ancillary data received. The *ancdata* item is a list of zero + or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing + the ancillary data (control messages) received: *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + :class:`bytes` object holding the associated data. The *msg_flags* + item is the bitwise OR of various flags indicating conditions on + the received message; see your system documentation for details. + If the receiving socket is unconnected, *address* is the address of + the sending socket, if available; otherwise, its value is + unspecified. + + On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to + pass file descriptors between processes over an :const:`AF_UNIX` + socket. When this facility is used (it is often restricted to + :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its + ancillary data, items of the form ``(socket.SOL_SOCKET, + socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object + representing the new file descriptors as a binary array of the + native C :c:expr:`int` type. If :meth:`recvmsg` raises an + exception after the system call returns, it will first attempt to + close any file descriptors received via this mechanism. + + Some systems do not indicate the truncated length of ancillary data + items which have been only partially received. If an item appears + to extend beyond the end of the buffer, :meth:`recvmsg` will issue + a :exc:`RuntimeWarning`, and will return the part of it which is + inside the buffer provided it has not been truncated before the + start of its associated data. + + On systems which support the :const:`!SCM_RIGHTS` mechanism, the + following function will receive up to *maxfds* file descriptors, + returning the message data and a list containing the descriptors + (while ignoring unexpected conditions such as unrelated control + messages being received). See also :meth:`sendmsg`. :: + + import socket, array + + def recv_fds(sock, msglen, maxfds): + fds = array.array("i") # Array of ints + msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: + # Append data, ignoring any truncated integers at the end. + fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + return msg, list(fds) + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + + + .. method:: recvmsg_into(buffers[, ancbufsize[, flags]]) + + Receive normal data and ancillary data from the socket, behaving as + :meth:`recvmsg` would, but scatter the non-ancillary data into a + series of buffers instead of returning a new bytes object. The + *buffers* argument must be an iterable of objects that export + writable buffers (e.g. :class:`bytearray` objects); these will be + filled with successive chunks of the non-ancillary data until it + has all been written or there are no more buffers. The operating + system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) + on the number of buffers that can be used. The *ancbufsize* and + *flags* arguments have the same meaning as for :meth:`recvmsg`. + + The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, + address)``, where *nbytes* is the total number of bytes of + non-ancillary data written into the buffers, and *ancdata*, + *msg_flags* and *address* are the same as for :meth:`recvmsg`. + + Example:: + + >>> import socket + >>> s1, s2 = socket.socketpair() + >>> b1 = bytearray(b'----') + >>> b2 = bytearray(b'0123456789') + >>> b3 = bytearray(b'--------------') + >>> s1.send(b'Mary had a little lamb') + 22 + >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) + (22, [], 0, None) + >>> [b1, b2, b3] + [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + + .. method:: recvfrom_into(buffer[, nbytes[, flags]]) + + Receive data from the socket, writing it into *buffer* instead of creating a + new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is + the number of bytes received and *address* is the address of the socket sending + the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the + optional argument *flags*; it defaults to zero. The format of *address* + depends on the address family --- see :ref:`socket-addresses`. + + + .. method:: recv_into(buffer[, nbytes[, flags]]) + + Receive up to *nbytes* bytes from the socket, storing the data into a buffer + rather than creating a new bytestring. If *nbytes* is not specified (or 0), + receive up to the size available in the given buffer. Returns the number of + bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning + of the optional argument *flags*; it defaults to zero. - Most Unix platforms. - .. versionadded:: 3.3 + .. method:: send(bytes[, flags]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Returns the number of bytes sent. Applications are responsible for checking that + all data has been sent; if only some of the data was transmitted, the + application needs to attempt delivery of the remaining data. For further + information on this topic, consult the :ref:`socket-howto`. -.. method:: socket.recvfrom_into(buffer[, nbytes[, flags]]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket, writing it into *buffer* instead of creating a - new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is - the number of bytes received and *address* is the address of the socket sending - the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. The format of *address* - depends on the address family --- see :ref:`socket-addresses`. + .. method:: sendall(bytes[, flags]) -.. method:: socket.recv_into(buffer[, nbytes[, flags]]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Unlike :meth:`send`, this method continues to send data from *bytes* until + either all data has been sent or an error occurs. ``None`` is returned on + success. On error, an exception is raised, and there is no way to determine how + much data, if any, was successfully sent. - Receive up to *nbytes* bytes from the socket, storing the data into a buffer - rather than creating a new bytestring. If *nbytes* is not specified (or 0), - receive up to the size available in the given buffer. Returns the number of - bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning - of the optional argument *flags*; it defaults to zero. + .. versionchanged:: 3.5 + The socket timeout is no longer reset each time data is sent successfully. + The socket timeout is now the maximum total duration to send all data. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). -.. method:: socket.send(bytes[, flags]) - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv`. - Returns the number of bytes sent. Applications are responsible for checking that - all data has been sent; if only some of the data was transmitted, the - application needs to attempt delivery of the remaining data. For further - information on this topic, consult the :ref:`socket-howto`. + .. method:: sendto(bytes, address) + sendto(bytes, flags, address) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send data to the socket. The socket should not be connected to a remote socket, + since the destination socket is specified by *address*. The optional *flags* + argument has the same meaning as for :meth:`recv`. Return the number of + bytes sent. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. + .. audit-event:: socket.sendto self,address socket.socket.sendto -.. method:: socket.sendall(bytes[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv`. - Unlike :meth:`send`, this method continues to send data from *bytes* until - either all data has been sent or an error occurs. ``None`` is returned on - success. On error, an exception is raised, and there is no way to determine how - much data, if any, was successfully sent. - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time data is sent successfully. - The socket timeout is now the maximum total duration to send all data. + .. method:: sendmsg(buffers[, ancdata[, flags[, address]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send normal and ancillary data to the socket, gathering the + non-ancillary data from a series of buffers and concatenating it + into a single message. The *buffers* argument specifies the + non-ancillary data as an iterable of + :term:`bytes-like objects ` + (e.g. :class:`bytes` objects); the operating system may set a limit + (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers + that can be used. The *ancdata* argument specifies the ancillary + data (control messages) as an iterable of zero or more tuples + ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + bytes-like object holding the associated data. Note that + some systems (in particular, systems without :func:`CMSG_SPACE`) + might support sending only one control message per call. The + *flags* argument defaults to 0 and has the same meaning as for + :meth:`send`. If *address* is supplied and not ``None``, it sets a + destination address for the message. The return value is the + number of bytes of non-ancillary data sent. + The following function sends the list of file descriptors *fds* + over an :const:`AF_UNIX` socket, on systems which support the + :const:`!SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: -.. method:: socket.sendto(bytes, address) - socket.sendto(bytes, flags, address) + import socket, array - Send data to the socket. The socket should not be connected to a remote socket, - since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv`. Return the number of - bytes sent. The format of *address* depends on the address family --- see - :ref:`socket-addresses`. + def send_fds(sock, msg, fds): + return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) - .. audit-event:: socket.sendto self,address socket.socket.sendto + .. availability:: Unix, not WASI. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]]) - - Send normal and ancillary data to the socket, gathering the - non-ancillary data from a series of buffers and concatenating it - into a single message. The *buffers* argument specifies the - non-ancillary data as an iterable of - :term:`bytes-like objects ` - (e.g. :class:`bytes` objects); the operating system may set a limit - (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers - that can be used. The *ancdata* argument specifies the ancillary - data (control messages) as an iterable of zero or more tuples - ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - bytes-like object holding the associated data. Note that - some systems (in particular, systems without :func:`CMSG_SPACE`) - might support sending only one control message per call. The - *flags* argument defaults to 0 and has the same meaning as for - :meth:`send`. If *address* is supplied and not ``None``, it sets a - destination address for the message. The return value is the - number of bytes of non-ancillary data sent. - - The following function sends the list of file descriptors *fds* - over an :const:`AF_UNIX` socket, on systems which support the - :const:`SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: - - import socket, array - - def send_fds(sock, msg, fds): - return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) + Most Unix platforms. - .. availability:: Unix, not WASI. + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg - Most Unix platforms. + .. versionadded:: 3.3 - .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. method:: sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. + Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. -.. method:: socket.sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) + .. availability:: Linux >= 2.6.38. - Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. - Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. + .. versionadded:: 3.6 - .. availability:: Linux >= 2.6.38. + .. method:: sendfile(file, offset=0, count=None) - .. versionadded:: 3.6 + Send a file until EOF is reached by using high-performance + :mod:`os.sendfile` and return the total number of bytes which were sent. + *file* must be a regular file object opened in binary mode. If + :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a + regular file :meth:`send` will be used instead. *offset* tells from where to + start reading the file. If specified, *count* is the total number of bytes + to transmit as opposed to sending the file until EOF is reached. File + position is updated on return or also in case of error in which case + :meth:`file.tell() ` can be used to figure out the number of + bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. + Non-blocking sockets are not supported. -.. method:: socket.sendfile(file, offset=0, count=None) + .. versionadded:: 3.5 - Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. - *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to - start reading the file. If specified, *count* is the total number of bytes - to transmit as opposed to sending the file until EOF is reached. File - position is updated on return or also in case of error in which case - :meth:`file.tell() ` can be used to figure out the number of - bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. - Non-blocking sockets are not supported. + .. method:: set_inheritable(inheritable) - .. versionadded:: 3.5 + Set the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle. -.. method:: socket.set_inheritable(inheritable) + .. versionadded:: 3.4 - Set the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle. - .. versionadded:: 3.4 + .. method:: setblocking(flag) + Set blocking or non-blocking mode of the socket: if *flag* is false, the + socket is set to non-blocking, else to blocking mode. -.. method:: socket.setblocking(flag) + This method is a shorthand for certain :meth:`~socket.settimeout` calls: - Set blocking or non-blocking mode of the socket: if *flag* is false, the - socket is set to non-blocking, else to blocking mode. + * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` - This method is a shorthand for certain :meth:`~socket.settimeout` calls: + * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` + .. versionchanged:: 3.7 + The method no longer applies :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - .. versionchanged:: 3.7 - The method no longer applies :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. + .. method:: settimeout(value) + Set a timeout on blocking socket operations. The *value* argument can be a + nonnegative real number expressing seconds, or ``None``. + If a non-zero value is given, subsequent socket operations will raise a + :exc:`timeout` exception if the timeout period *value* has elapsed before + the operation has completed. If zero is given, the socket is put in + non-blocking mode. If ``None`` is given, the socket is put in blocking mode. -.. method:: socket.settimeout(value) + For further information, please consult the :ref:`notes on socket timeouts `. - Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative real number expressing seconds, or ``None``. - If a non-zero value is given, subsequent socket operations will raise a - :exc:`timeout` exception if the timeout period *value* has elapsed before - the operation has completed. If zero is given, the socket is put in - non-blocking mode. If ``None`` is given, the socket is put in blocking mode. + .. versionchanged:: 3.7 + The method no longer toggles :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - For further information, please consult the :ref:`notes on socket timeouts `. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. - .. versionchanged:: 3.7 - The method no longer toggles :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. - .. versionchanged:: 3.15 - Accepts any real number, not only integer or float. + .. method:: setsockopt(level, optname, value: int | Buffer) + setsockopt(level, optname, None, optlen: int) + .. index:: pair: module; struct -.. method:: socket.setsockopt(level, optname, value: int | Buffer) - socket.setsockopt(level, optname, None, optlen: int) + Set the value of the given socket option (see the Unix manual page + :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this + module (:ref:`!SO_\* etc. `). The value can be an integer, + ``None`` or a :term:`bytes-like object` representing a buffer. In the latter + case it is up to the caller to ensure that the bytestring contains the + proper bits (see the optional built-in module :mod:`struct` for a way to + encode C structures as bytestrings). When *value* is set to ``None``, + *optlen* argument is required. It's equivalent to calling :c:func:`!setsockopt` C + function with ``optval=NULL`` and ``optlen=optlen``. - .. index:: pair: module; struct + .. versionchanged:: 3.5 + Writable :term:`bytes-like object` is now accepted. - Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this - module (:ref:`!SO_\* etc. `). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the latter - case it is up to the caller to ensure that the bytestring contains the - proper bits (see the optional built-in module :mod:`struct` for a way to - encode C structures as bytestrings). When *value* is set to ``None``, - *optlen* argument is required. It's equivalent to calling :c:func:`setsockopt` C - function with ``optval=NULL`` and ``optlen=optlen``. + .. versionchanged:: 3.6 + setsockopt(level, optname, None, optlen: int) form added. - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. + .. availability:: not WASI. - .. versionchanged:: 3.6 - setsockopt(level, optname, None, optlen: int) form added. - .. availability:: not WASI. + .. method:: shutdown(how) + Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, + further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends + are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are + disallowed. -.. method:: socket.shutdown(how) + .. availability:: not WASI. - Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, - further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends - are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are - disallowed. - .. availability:: not WASI. + .. method:: share(process_id) + Duplicate a socket and prepare it for sharing with a target process. The + target process must be provided with *process_id*. The resulting bytes object + can then be passed to the target process using some form of interprocess + communication and the socket can be recreated there using :func:`fromshare`. + Once this method has been called, it is safe to close the socket since + the operating system has already duplicated it for the target process. -.. method:: socket.share(process_id) + .. availability:: Windows. - Duplicate a socket and prepare it for sharing with a target process. The - target process must be provided with *process_id*. The resulting bytes object - can then be passed to the target process using some form of interprocess - communication and the socket can be recreated there using :func:`fromshare`. - Once this method has been called, it is safe to close the socket since - the operating system has already duplicated it for the target process. + .. versionadded:: 3.3 - .. availability:: Windows. - .. versionadded:: 3.3 + Note that there are no methods :meth:`!read` or :meth:`!write`; use + :meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. + Socket objects also have these (read-only) attributes that correspond to the + values given to the :class:`~socket.socket` constructor. -Note that there are no methods :meth:`read` or :meth:`write`; use -:meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. -Socket objects also have these (read-only) attributes that correspond to the -values given to the :class:`~socket.socket` constructor. + .. attribute:: family + The socket family. -.. attribute:: socket.family - The socket family. + .. attribute:: type + The socket type. -.. attribute:: socket.type - The socket type. + .. attribute:: proto + The socket protocol. -.. attribute:: socket.proto - The socket protocol. +.. class:: SocketType + The base class of the :class:`~socket.socket` type, re-exported from + :mod:`!_socket`. An instance check such as + ``isinstance(socket(...), SocketType)`` is true, but ``SocketType`` is not + the same as ``type(socket(...))``, which is :class:`~socket.socket` itself. .. _socket-timeouts: diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 484260e63dd5f2f..3a75d44f3f7d21b 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -1417,6 +1417,9 @@ Connection objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: next + Deleting the ``row_factory`` attribute is no longer allowed. + .. attribute:: text_factory A :term:`callable` that accepts a :class:`bytes` parameter @@ -1426,6 +1429,9 @@ Connection objects See :ref:`sqlite3-howto-encoding` for more details. + .. versionchanged:: next + Deleting the ``text_factory`` attribute is no longer allowed. + .. attribute:: total_changes Return the total number of database rows that have been modified, inserted, or @@ -1709,6 +1715,9 @@ Cursor objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: next + Deleting the ``row_factory`` attribute is no longer allowed. + .. The sqlite3.Row example used to be a how-to. It has now been incorporated into the Row reference. We keep the anchor here in order not to break diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index f2c35d1897a77f8..66fe6c7aee48626 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -146,9 +146,9 @@ purposes. *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load default CA certificates. - When :attr:`~SSLContext.keylog_filename` is supported and the environment - variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` - enables key logging. + When the environment variable :envvar:`!SSLKEYLOGFILE` is set, + :func:`create_default_context` enables key logging by setting + :attr:`~SSLContext.keylog_filename` to the variable's value. The default settings for this context include :data:`VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT`. @@ -1121,7 +1121,7 @@ SSL sockets :meth:`SSLContext.wrap_socket` to wrap a socket. .. versionchanged:: 3.7 - :class:`SSLSocket` instances must to created with + :class:`SSLSocket` instances must be created with :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. @@ -2076,7 +2076,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2879,11 +2879,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 3d943566be34ff1..886648e820f071d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1403,6 +1403,8 @@ application). Many other operations also produce lists, including the :func:`sorted` built-in. + Lists are :ref:`generic ` over the types of their items. + Lists implement all of the :ref:`common ` and :ref:`mutable ` sequence operations. Lists also provide the following additional method: @@ -1494,6 +1496,10 @@ homogeneous data is needed (such as allowing storage in a :class:`set` or Tuples implement all of the :ref:`common ` sequence operations. + Tuples are :ref:`generic ` over the types of their contents. + For more information, refer to + :ref:`the typing documentation on annotating tuples `. + For heterogeneous collections of data where access by name is clearer than access by index, :func:`collections.namedtuple` may be a more appropriate choice than a simple tuple object. @@ -2168,9 +2174,25 @@ expression support in the :mod:`re` module). character, ``False`` otherwise. Digits include decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the + like the `Kharosthi numbers `__. + Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + .. doctest:: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isnumeric`. + .. method:: str.isidentifier() @@ -2211,15 +2233,14 @@ expression support in the :mod:`re` module). >>> '0123456789'.isnumeric() True - >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine True >>> '⅕'.isnumeric() # Vulgar fraction one fifth True >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() (False, True, True) - See also :meth:`isdecimal` and :meth:`isdigit`. Numeric characters are - a superset of decimal numbers. + See also :meth:`isdecimal` and :meth:`isdigit`. .. method:: str.isprintable() @@ -2370,7 +2391,8 @@ expression support in the :mod:`re` module). Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* + or ``None``, the *chars* argument defaults to removing whitespace, that is + characters for which :meth:`str.isspace` is true. The *chars* argument is not a prefix; rather, all combinations of its values are stripped:: >>> ' spacious '.lstrip() @@ -2573,7 +2595,8 @@ expression support in the :mod:`re` module). Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* + or ``None``, the *chars* argument defaults to removing whitespace, that is + characters for which :meth:`str.isspace` is true. The *chars* argument is not a suffix; rather, all combinations of its values are stripped. For example: @@ -2610,7 +2633,9 @@ expression support in the :mod:`re` module). :func:`re.split`). Splitting an empty string with a specified separator returns ``['']``. - For example:: + For example: + + .. doctest:: >>> '1,2,3'.split(',') ['1', '2', '3'] @@ -2628,7 +2653,9 @@ expression support in the :mod:`re` module). string or a string consisting of just whitespace with a ``None`` separator returns ``[]``. - For example:: + For example: + + .. doctest:: >>> '1 2 3'.split() ['1', '2', '3'] @@ -2640,7 +2667,9 @@ expression support in the :mod:`re` module). If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only leading runs of consecutive whitespace are considered. - For example:: + For example: + + .. doctest:: >>> "".split(None, 0) [] @@ -2649,7 +2678,7 @@ expression support in the :mod:`re` module). >>> " foo ".split(maxsplit=0) ['foo '] - See also :meth:`join`. + See also :meth:`join` and :meth:`rsplit`. .. index:: @@ -2743,9 +2772,9 @@ expression support in the :mod:`re` module). Return a copy of the string with the leading and trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. - If omitted or ``None``, the *chars* argument defaults to removing whitespace. - The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped. + If omitted or ``None``, the *chars* argument defaults to removing whitespace, + that is characters for which :meth:`str.isspace` is true. The *chars* argument + is not a prefix or suffix; rather, all combinations of its values are stripped. For example: @@ -4584,6 +4613,9 @@ copying. types such as :class:`bytes` and :class:`bytearray`, an element is a single byte, but other types such as :class:`array.array` may have bigger elements. + :class:`!memoryview`\s are :ref:`generic ` over the type of their + underlying data. + ``len(view)`` is equal to the length of :meth:`~memoryview.tolist`, which is the nested list representation of the view. If ``view.ndim = 1``, this is equal to the number of elements in the view. @@ -5277,6 +5309,8 @@ Note, the *elem* argument to the :meth:`~object.__contains__`, :meth:`~set.discard` methods may be a set. To support searching for an equivalent frozenset, a temporary one is created from *elem*. +Sets and frozensets are :ref:`generic ` over the type of their elements. + .. seealso:: For detailed information on thread-safety guarantees for :class:`set` @@ -5380,6 +5414,9 @@ can be used interchangeably to index the same dictionary entry. Dictionary order is guaranteed to be insertion order. This behavior was an implementation detail of CPython from 3.6. + Dictionaries are :ref:`generic ` over two types, signifying + (respectively) the types of the dictionary's keys and values. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -5717,6 +5754,16 @@ Frozen dictionaries :class:`!frozendict` is not a :class:`!dict` subclass but inherits directly from ``object``. + Like dictionaries, frozendicts are :ref:`generic ` over two types, + signifying (respectively) the types of the frozendict's keys and values. + + .. classmethod:: fromkeys(iterable, value=None, /) + + Similar to :meth:`dict.fromkeys`, but call again the type constructor + with an initialized :class:`frozendict` if the type is a + :class:`frozendict` subclass or if the constructor returned a + :class:`frozendict`. + .. versionadded:: 3.15 @@ -5858,7 +5905,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] @@ -5907,6 +5955,15 @@ creation:: >>> type(l) + +Instances of ``GenericAlias`` are not classes at runtime, even though they behave like classes (they can be instantiated and subclassed):: + + >>> import inspect + >>> inspect.isclass(list[int]) + False + +This is true for :ref:`user-defined generics ` also. + Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: >>> repr(list[int]) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 08ccdfa3f454f8d..be968a3c53d8430 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -472,7 +472,9 @@ of a number respectively. It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -518,9 +520,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -603,10 +610,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index a5b2d796aaf4b83..6e91850fdf59ca4 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -245,6 +245,8 @@ Types ... 3.0 | 1. + 2. | None | .2f + Interpolations are :ref:`generic ` over the types of their values. + .. rubric:: Attributes .. attribute:: value diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index bf5f754e156d3c8..ec872fddee05c70 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -111,7 +111,7 @@ Format Strings -------------- Format strings describe the data layout when -packing and unpacking data. They are built up from :ref:`format characters`, +packing and unpacking data. They are built up from :ref:`type codes `, which specify the type of data being packed/unpacked. In addition, special characters control the :ref:`byte order, size and alignment`. Each format string consists of an optional prefix character which @@ -183,8 +183,8 @@ Use :data:`sys.byteorder` to check the endianness of your system. Native size and alignment are determined using the C compiler's ``sizeof`` expression. This is always combined with native byte order. -Standard size depends only on the format character; see the table in -the :ref:`format-characters` section. +Standard size depends only on the type code; see the table in +the :ref:`type-codes` section. Note the difference between ``'@'`` and ``'='``: both use native byte order, but the size and alignment of the latter is standardized. @@ -208,12 +208,13 @@ Notes: count of zero. See :ref:`struct-examples`. +.. _type-codes: .. _format-characters: -Format Characters -^^^^^^^^^^^^^^^^^ +Type Codes +^^^^^^^^^^ -Format characters have the following meaning; the conversion between C and +Type codes (or format codes) have the following meaning; the conversion between C and Python values should be obvious given their types. The 'Standard size' column refers to the size of the packed value in bytes when using standard size; that is, when the format string starts with one of ``'<'``, ``'>'``, ``'!'`` or @@ -284,7 +285,7 @@ platform-dependent. .. versionchanged:: 3.14 Added support for the ``'F'`` and ``'D'`` formats. -.. versionchanged:: next +.. versionchanged:: 3.15 Added support for the ``'Zf'`` and ``'Zd'`` formats. .. seealso:: @@ -324,7 +325,7 @@ Notes: format used by the platform. (5) - The ``'P'`` format character is only available for the native byte ordering + The ``'P'`` type code is only available for the native byte ordering (selected as the default or with the ``'@'`` byte order character). The byte order character ``'='`` chooses to use little- or big-endian ordering based on the host system. The struct module does not interpret this as native @@ -346,22 +347,22 @@ Notes: When packing, ``'x'`` inserts one NUL byte. (8) - The ``'p'`` format character encodes a "Pascal string", meaning a short + The ``'p'`` type code encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all - are used. Note that for :func:`unpack`, the ``'p'`` format character consumes + are used. Note that for :func:`unpack`, the ``'p'`` type code consumes ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. When packing, arguments of types :class:`bytes` and :class:`bytearray` are accepted. (9) - For the ``'s'`` format character, the count is interpreted as the length of the - byte string, not a repeat count like for the other format characters; for example, + For the ``'s'`` type code, the count is interpreted as the length of the + byte string, not a repeat count like for the other type codes; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping @@ -376,7 +377,7 @@ Notes: are accepted. (10) - For the ``'F'`` and ``'D'`` format characters, the packed representation uses + For the ``'F'`` and ``'D'`` type codes, the packed representation uses the IEEE 754 binary32 and binary64 format for components of the complex number, regardless of the floating-point format used by the platform. Note that complex types (``F``/``Zf`` and ``D``/``Zd``) are available unconditionally, @@ -385,7 +386,7 @@ Notes: two-element C array containing, respectively, the real and imaginary parts. -A format character may be preceded by an integral repeat count. For example, +A type code may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. Whitespace characters between formats are ignored; a count and its format must @@ -402,7 +403,7 @@ then :exc:`struct.error` is raised. .. index:: single: ? (question mark); in struct format strings -For the ``'?'`` format character, the return value is either :const:`True` or +For the ``'?'`` type code, the return value is either :const:`True` or :const:`False`. When packing, the truth value of the argument object is used. Either 0 or 1 in the native or standard bool representation will be packed, and any non-zero value will be ``True`` when unpacking. @@ -457,7 +458,7 @@ the result in a named tuple:: >>> Student._make(unpack('<10sHHb', record)) Student(name=b'raymond ', serialnum=4658, school=264, gradelevel=8) -The ordering of format characters may have an impact on size in native +The ordering of type codes may have an impact on size in native mode since padding is implicit. In standard mode, the user is responsible for inserting any desired padding. Note in @@ -515,7 +516,7 @@ When constructing format strings which mimic native layouts, the compiler and machine architecture determine byte ordering and padding. In such cases, the ``@`` format character should be used to specify native byte ordering and data sizes. Internal pad bytes are normally inserted -automatically. It is possible that a zero-repeat format code will be +automatically. It is possible that a zero-repeat type code will be needed at the end of a format string to round up to the correct byte boundary for proper alignment of consecutive chunks of data. @@ -534,7 +535,7 @@ code solves that problem:: >>> calcsize('@llh0l') 24 -The ``'x'`` format code can be used to specify the repeat, but for +The ``'x'`` type code can be used to specify the repeat, but for native formats it is better to use a zero-repeat format like ``'0l'``. By default, native byte ordering and alignment is used, but it is diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 52a722608db4312..95f20b06b5aa1e7 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -187,57 +187,6 @@ Examining Symbol Tables A namespace of a class. This class inherits from :class:`SymbolTable`. - .. method:: get_methods() - - Return a tuple containing the names of method-like functions declared - in the class. - - Here, the term 'method' designates *any* function defined in the class - body via :keyword:`def` or :keyword:`async def`. - - Functions defined in a deeper scope (e.g., in an inner class) are not - picked up by :meth:`get_methods`. - - For example: - - .. testsetup:: symtable.Class.get_methods - - import warnings - context = warnings.catch_warnings() - context.__enter__() - warnings.simplefilter("ignore", category=DeprecationWarning) - - .. testcleanup:: symtable.Class.get_methods - - context.__exit__() - - .. doctest:: symtable.Class.get_methods - - >>> import symtable - >>> st = symtable.symtable(''' - ... def outer(): pass - ... - ... class A: - ... def f(): - ... def w(): pass - ... - ... def g(self): pass - ... - ... @classmethod - ... async def h(cls): pass - ... - ... global outer - ... def outer(self): pass - ... ''', 'test', 'exec') - >>> class_A = st.get_children()[2] - >>> class_A.get_methods() - ('f', 'g', 'h') - - Although ``A().f()`` raises :exc:`TypeError` at runtime, ``A.f`` is still - considered as a method-like function. - - .. deprecated-removed:: 3.14 3.16 - .. class:: Symbol diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 6946eb6eeaa5fae..d0fe0625deb5133 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -879,7 +879,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionchanged:: 3.6 Windows is no longer guaranteed to return ``'mbcs'``. See :pep:`529` - and :func:`_enablelegacywindowsfsencoding` for more information. + for more information. .. versionchanged:: 3.7 Return ``'utf-8'`` if the :ref:`Python UTF-8 Mode ` is @@ -919,8 +919,6 @@ always available. Unless explicitly noted otherwise, all variables are read-only * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword are lazy * ``"all"``: All top-level imports are potentially lazy - * ``"none"``: All lazy imports are suppressed (even explicitly marked - ones) See also :func:`set_lazy_imports` and :pep:`810`. @@ -1483,6 +1481,21 @@ always available. Unless explicitly noted otherwise, all variables are read-only They hold the legacy representation of ``sys.last_exc``, as returned from :func:`exc_info` above. + +.. data:: lazy_modules + + A :class:`set` of fully qualified module name strings that have been lazily + imported in the current interpreter but not yet loaded. When a + lazily imported module is accessed for the first time, its name is removed + from this set. + + This attribute is intended for debugging and introspection. + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + .. data:: maxsize An integer giving the maximum value a variable of type :c:type:`Py_ssize_t` can @@ -1757,8 +1770,6 @@ always available. Unless explicitly noted otherwise, all variables are read-only * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword are lazy * ``"all"``: All top-level imports become potentially lazy - * ``"none"``: All lazy imports are suppressed (even explicitly marked - ones) This function is intended for advanced users who need to control lazy imports across their entire application. Library developers should @@ -2107,31 +2118,6 @@ always available. Unless explicitly noted otherwise, all variables are read-only See :pep:`768` for more details. -.. function:: _enablelegacywindowsfsencoding() - - Changes the :term:`filesystem encoding and error handler` to 'mbcs' and - 'replace' respectively, for consistency with versions of Python prior to - 3.6. - - This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` - environment variable before launching Python. - - See also :func:`sys.getfilesystemencoding` and - :func:`sys.getfilesystemencodeerrors`. - - .. availability:: Windows. - - .. note:: - Changing the filesystem encoding after Python startup is risky because - the old fsencoding or paths encoded by the old fsencoding may be cached - somewhere. Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. - - .. versionadded:: 3.6 - See :pep:`529` for more details. - - .. deprecated-removed:: 3.13 3.16 - Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. - .. data:: stdin stdout stderr diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 6f1e01cf5aa6ee9..29a329fdfeab15b 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -144,7 +144,9 @@ Some facts and figures: For modes ``'w:gz'`` and ``'w|gz'``, :func:`tarfile.open` accepts the keyword argument *mtime* to create a gzip archive header with that mtime. By - default, the mtime is set to the time of creation of the archive. + default, the mtime is set to the time of creation of the archive. Use + *mtime* ``0`` to generate a compressed stream that does not depend on + creation time, for reproducible output. For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` @@ -668,7 +670,7 @@ be finalized; only the internally used file object will be closed. See the it is best practice to only do so in top-level applications or :mod:`site configuration `. To set a global default this way, a filter function needs to be wrapped in - :func:`staticmethod` to prevent injection of a ``self`` argument. + :deco:`staticmethod` to prevent injection of a ``self`` argument. .. versionchanged:: 3.14 diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 7ae3fabf1cec644..4e21e1ded82724c 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -718,7 +718,7 @@ The :mod:`!test.support` module defines the following functions: .. decorator:: anticipate_failure(condition) A decorator to conditionally mark tests with - :func:`unittest.expectedFailure`. Any use of this decorator should + :deco:`unittest.expectedFailure`. Any use of this decorator should have an associated comment identifying the relevant tracker issue. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index fbe3951e034d072..5d9a7b6314b1668 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -515,7 +515,7 @@ since it is impossible to detect the termination of alien threads. This constructor should always be called with keyword arguments. Arguments are: - *group* should be ``None``; reserved for future extension when a + *group* must be ``None`` as it is reserved for future extension when a :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. @@ -1502,7 +1502,7 @@ of the derived iterators. In this example, each number is printed exactly once, but the work is shared between the two threads. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: synchronized_iterator(func) @@ -1540,7 +1540,7 @@ of the derived iterators. The returned wrapper preserves the metadata of *func*, such as its name and wrapped function reference. - .. versionadded:: next + .. versionadded:: 3.15 .. function:: concurrent_tee(iterable, n=2) @@ -1591,4 +1591,4 @@ of the derived iterators. In this example, both consumer threads see the full sequence of squares from a single generator expression. - .. versionadded:: next + .. versionadded:: 3.15 diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index fd67c5c0a0f5133..dd574fce09f5fc8 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -19,7 +19,7 @@ See also Tim Peters' introduction to the "Algorithms" chapter in the second edition of *Python Cookbook*, published by O'Reilly. -Basic Examples +Basic examples -------------- The following example shows how the :ref:`timeit-command-line-interface` @@ -56,7 +56,7 @@ repetitions only when the command-line interface is used. In the .. _python-interface: -Python Interface +Python interface ---------------- The module defines three convenience functions and a public class: @@ -206,7 +206,7 @@ The module defines three convenience functions and a public class: .. _timeit-command-line-interface: -Command-Line Interface +Command-line interface ---------------------- When called as a program from the command line, the following form is used:: @@ -279,6 +279,9 @@ most cases. You can use :func:`time.process_time` to measure CPU time. baseline overhead can be measured by invoking the program without arguments, and it might differ between Python versions. +.. versionadded:: 3.15 + Output is in color by default and can be + :ref:`controlled using environment variables `. .. _timeit-examples: diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst index 73f8f76a21044b0..e1e04d04a08b80e 100644 --- a/Doc/library/tkinter.colorchooser.rst +++ b/Doc/library/tkinter.colorchooser.rst @@ -15,14 +15,26 @@ the :class:`~tkinter.commondialog.Dialog` class. .. class:: Chooser(master=None, **options) + The class implementing the modal color-choosing dialog. + Most applications use the :func:`askcolor` convenience function rather than + instantiating this class directly. + .. function:: askcolor(color=None, **options) - Create a color choosing dialog. A call to this method will show the window, - wait for the user to make a selection, and return the selected color (or - ``None``) to the caller. + Show a modal color-choosing dialog and return the chosen color. + *color* is the color selected when the dialog opens. + The return value is a tuple ``((r, g, b), hexstr)``, where ``r``, ``g`` and + ``b`` are the red, green and blue components as integers in the range 0–255 + and *hexstr* is the equivalent Tk color string, such as ``'#ff8000'``. + If the user cancels the dialog, ``(None, None)`` is returned. + + .. versionchanged:: 3.10 + The RGB values in the returned color are now integers in the range 0–255 + instead of floats. .. seealso:: Module :mod:`tkinter.commondialog` Tkinter standard dialog module + diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst index 48d16ccb204b9d5..ba267c0e3840fe2 100644 --- a/Doc/library/tkinter.dnd.rst +++ b/Doc/library/tkinter.dnd.rst @@ -48,7 +48,8 @@ Selection of a target object occurs as follows: .. method:: on_motion(event) - Inspect area below mouse for target objects while drag is performed. + Inspect area below mouse for target objects while a drag + is performed. .. method:: on_release(event) @@ -56,7 +57,9 @@ Selection of a target object occurs as follows: .. function:: dnd_start(source, event) - Factory function for drag-and-drop process. + Factory function for the drag-and-drop process. + Return the :class:`DndHandler` instance managing the drag, or ``None`` if a + drag could not be started. .. seealso:: diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index 9eecb803c3aedcb..2a9563fffab1f71 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -26,6 +26,10 @@ The different font weights and slants are: fonts as a single object, rather than specifying a font by its attributes with each occurrence. + .. versionchanged:: 3.10 + Two fonts now compare equal (``==``) only when both are :class:`Font` + instances with the same name belonging to the same Tcl interpreter. + arguments: | *font* - font specifier tuple (family, size, options) @@ -34,7 +38,7 @@ The different font weights and slants are: additional keyword options (ignored if *font* is specified): - | *family* - font family i.e. Courier, Times + | *family* - font family, for example, Courier, Times | *size* - font size | If *size* is positive it is interpreted as size in points. | If *size* is a negative number its absolute value is treated @@ -46,15 +50,24 @@ The different font weights and slants are: .. method:: actual(option=None, displayof=None) - Return the attributes of the font. + Return the actual attributes of the font, which may differ from the + requested ones because of platform limitations. + With no *option*, return a dictionary of all the attributes; if *option* + is given, return the value of that single attribute. .. method:: cget(option) Retrieve an attribute of the font. .. method:: config(**options) + :no-typesetting: + + .. method:: configure(**options) + + Modify one or more attributes of the font. + With no arguments, return a dictionary of the current attributes. - Modify attributes of the font. + :meth:`config` is an alias of :meth:`!configure`. .. method:: copy() @@ -63,12 +76,15 @@ The different font weights and slants are: .. method:: measure(text, displayof=None) Return amount of space the text would occupy on the specified display - when formatted in the current font. If no display is specified then the - main application window is assumed. + when formatted in the current font, as an integer number of pixels. + If no display is specified then the main application window is assumed. .. method:: metrics(*options, **kw) Return font-specific data. + With no options, return a dictionary mapping each metric name to its + integer value; if one option name is given, return that metric's value as + an integer. Options include: *ascent* - distance between baseline and highest point that a @@ -84,15 +100,17 @@ The different font weights and slants are: .. function:: families(root=None, displayof=None) - Return the different font families. + Return a tuple of the names of the available font families. .. function:: names(root=None) - Return the names of defined fonts. + Return a tuple of the names of all the defined fonts. .. function:: nametofont(name, root=None) - Return a :class:`Font` representation of a tk named font. + Return a :class:`Font` representation of the existing named font *name*. + *root* is the widget whose Tcl interpreter owns the font; if omitted, the + default root window is used. .. versionchanged:: 3.10 The *root* parameter was added. diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 2a69d282638529d..b4529e2329ebfbc 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -9,11 +9,14 @@ -------------- The :mod:`!tkinter.messagebox` module provides a template base class as well as -a variety of convenience methods for commonly used configurations. The message -boxes are modal and will return a subset of (``True``, ``False``, ``None``, -:data:`OK`, :data:`CANCEL`, :data:`YES`, :data:`NO`) based on -the user's selection. Common message box styles and layouts include but are not -limited to: +a variety of convenience methods for commonly used configurations. +The message boxes are modal: each blocks until the user responds, then returns +a value that depends on the function. +The ``show*`` functions and :meth:`Message.show` return the symbolic name of +the button the user pressed, as a string (such as :data:`OK` or :data:`YES`), +while the ``ask*`` functions return a :class:`bool` or ``None`` (see each +function below). +Common message box styles and layouts include but are not limited to: .. figure:: tk_msg.png @@ -66,6 +69,10 @@ limited to: Arranges for a :ref:`predefined set of buttons ` to be displayed. + .. note:: + + Tk 8.6 added the *command* option. + .. method:: show(**options) Display a message window and wait for the user to select one of the buttons. Then return the symbolic name of the selected button. diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index a34b74a088874fe..5fe1ed7474cf367 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -17,10 +17,15 @@ demonstrating a simple Tk interface, letting you know that :mod:`!tkinter` is properly installed on your system, and also showing what version of Tcl/Tk is installed, so you can read the Tcl/Tk documentation specific to that version. -Tkinter supports a range of Tcl/Tk versions, built either with or -without thread support. The official Python binary release bundles Tcl/Tk 8.6 -threaded. See the source code for the :mod:`_tkinter` module -for more information about supported versions. +Tkinter supports a range of Tcl/Tk versions, built either with or without +thread support. +Tcl/Tk 8.5.12 is the minimum supported version; the official Python binary +release bundles Tcl/Tk 9.0. +See the source code for the :mod:`_tkinter` module for more information about +supported versions. + +.. versionchanged:: 3.11 + Support for Tcl/Tk versions older than 8.5.12 was removed. Tkinter is not a thin wrapper, but adds a fair amount of its own logic to make the experience more pythonic. This documentation will concentrate on these @@ -47,7 +52,7 @@ details that are unchanged. Tcl/Tk Resources: - * `Tk commands `_ + * `Tk commands `_ Comprehensive reference to each of the underlying Tcl/Tk commands used by Tkinter. * `Tcl/Tk Home Page `_ @@ -103,16 +108,16 @@ Ttk bindings are provided in a separate module, :mod:`tkinter.ttk`. Internally, Tk and Ttk use facilities of the underlying operating system, -i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. +that is, Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. -When your Python application uses a class in Tkinter, e.g., to create a widget, +When your Python application uses a class in Tkinter, for example, to create a widget, the :mod:`!tkinter` module first assembles a Tcl/Tk command string. It passes that Tcl command string to an internal :mod:`_tkinter` binary module, which then calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. -Tkinter Modules +Tkinter modules --------------- Support for Tkinter is spread across several modules. Most applications will need the @@ -124,82 +129,6 @@ the modern themed widget set and API:: from tkinter import ttk -.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) - - Construct a toplevel Tk widget, which is usually the main window of an - application, and initialize a Tcl interpreter for this widget. Each - instance has its own associated Tcl interpreter. - - The :class:`Tk` class is typically instantiated using all default values. - However, the following keyword arguments are currently recognized: - - *screenName* - When given (as a string), sets the :envvar:`DISPLAY` environment - variable. (X11 only) - *baseName* - Name of the profile file. By default, *baseName* is derived from the - program name (``sys.argv[0]``). - *className* - Name of the widget class. Used as a profile file and also as the name - with which Tcl is invoked (*argv0* in *interp*). - *useTk* - If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` - function sets this to ``False``. - *sync* - If ``True``, execute all X server commands synchronously, so that errors - are reported immediately. Can be used for debugging. (X11 only) - *use* - Specifies the *id* of the window in which to embed the application, - instead of it being created as an independent toplevel window. *id* must - be specified in the same way as the value for the -use option for - toplevel widgets (that is, it has a form like that returned by - :meth:`winfo_id`). - - Note that on some platforms this will only work correctly if *id* refers - to a Tk frame or toplevel that has its -container option enabled. - - :class:`Tk` reads and interprets profile files, named - :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl - interpreter and calls :func:`exec` on the contents of - :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the - profile files is the :envvar:`HOME` environment variable or, if that - isn't defined, then :data:`os.curdir`. - - .. attribute:: tk - - The Tk application object created by instantiating :class:`Tk`. This - provides access to the Tcl interpreter. Each widget that is attached - the same instance of :class:`Tk` has the same value for its :attr:`tk` - attribute. - - .. attribute:: master - - The widget object that contains this widget. For :class:`Tk`, the - :attr:`!master` is :const:`None` because it is the main window. The terms - *master* and *parent* are similar and sometimes used interchangeably - as argument names; however, calling :meth:`winfo_parent` returns a - string of the widget name whereas :attr:`!master` returns the object. - *parent*/*child* reflects the tree-like relationship while - *master* (or *container*)/*content* reflects the container structure. - - .. attribute:: children - - The immediate descendants of this widget as a :class:`dict` with the - child widget names as the keys and the child instance objects as the - values. - - -.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) - - The :func:`Tcl` function is a factory function which creates an object much like - that created by the :class:`Tk` class, except that it does not initialize the Tk - subsystem. This is most often useful when driving the Tcl interpreter in an - environment where one doesn't want to create extraneous toplevel windows, or - where one cannot (such as Unix/Linux systems without an X server). An object - created by the :func:`Tcl` object can have a Toplevel window created (and the Tk - subsystem initialized) by calling its :meth:`loadtk` method. - - The modules that provide Tk support include: :mod:`!tkinter` @@ -246,7 +175,7 @@ Additional modules: Python's Integrated Development and Learning Environment (IDLE). Based on :mod:`!tkinter`. -:mod:`tkinter.constants` +:mod:`!tkinter.constants` Symbolic constants that can be used in place of strings when passing various parameters to Tkinter calls. Automatically imported by the main :mod:`!tkinter` module. @@ -258,8 +187,10 @@ Additional modules: :mod:`turtle` Turtle graphics in a Tk window. +.. currentmodule:: tkinter -Tkinter Life Preserver + +Tkinter life preserver ---------------------- This section is not designed to be an exhaustive tutorial on either Tk or @@ -274,7 +205,7 @@ find more detailed documentation on them, including in the official Tcl/Tk reference manual. -A Hello World Program +A Hello World program ^^^^^^^^^^^^^^^^^^^^^ We'll start by walking through a "Hello World" application in Tkinter. This @@ -302,19 +233,20 @@ The following line creates a frame widget, which in this case will contain a label and a button we'll create next. The frame is fit inside the root window. -The next line creates a label widget holding a static text string. The -:meth:`grid` method is used to specify the relative layout (position) of the -label within its containing frame widget, similar to how tables in HTML work. +The next line creates a label widget holding a static text string. +The :meth:`~Grid.grid` method is used to specify the relative layout (position) +of the label within its containing frame widget, similar to how tables in HTML +work. -A button widget is then created, and placed to the right of the label. When -pressed, it will call the :meth:`destroy` method of the root window. +A button widget is then created, and placed to the right of the label. +When pressed, it will call the :meth:`~Misc.destroy` method of the root window. Finally, the :meth:`mainloop` method puts everything on the display, and responds to user input until the program terminates. -Important Tk Concepts +Important Tk concepts ^^^^^^^^^^^^^^^^^^^^^ Even this simple program illustrates the following key Tk concepts: @@ -346,7 +278,7 @@ event loop isn't running the event loop, your user interface won't update. -Understanding How Tkinter Wraps Tcl/Tk +Understanding how Tkinter wraps Tcl/Tk ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When your application uses Tkinter's classes and methods, internally Tkinter @@ -414,9 +346,9 @@ interactive Python shell or with :func:`print`, can help you identify what you need. To find out what configuration options are available on any widget, call its -:meth:`configure` method, which returns a dictionary containing a variety of -information about each object, including its default and current values. Use -:meth:`keys` to get just the names of each option. +:meth:`~Misc.configure` method, which returns a dictionary containing a variety +of information about each object, including its default and current values. +Use :meth:`~Misc.keys` to get just the names of each option. :: @@ -443,18 +375,20 @@ is helpful. print(set(dir(btn)) - set(dir(frm))) -Navigating the Tcl/Tk Reference Manual +Navigating the Tcl/Tk reference manual ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As noted, the official `Tk commands `_ -reference manual (man pages) is often the most accurate description of what -specific operations on widgets do. Even when you know the name of the option -or method that you need, you may still have a few places to look. +As noted, the official +`Tk commands `_ reference +manual (man pages) is often the most accurate description of what specific +operations on widgets do. +Even when you know the name of the option or method that you need, you may +still have a few places to look. While all operations in Tkinter are implemented as method calls on widget objects, you've seen that many Tcl/Tk operations appear as commands that take a widget pathname as its first parameter, followed by optional -parameters, e.g. +parameters, for example :: @@ -473,22 +407,23 @@ name of a method to call). In the official Tcl/Tk reference documentation, you'll find most operations -that look like method calls on the man page for a specific widget (e.g., -you'll find the :meth:`invoke` method on the -`ttk::button `_ +that look like method calls on the man page for a specific widget (for example, +you'll find the :meth:`~tkinter.ttk.Button.invoke` method on the +`ttk::button `_ man page), while functions that take a widget as a parameter often have -their own man page (e.g., -`grid `_). +their own man page (for example, +`grid `_). You'll find many common options and methods in the -`options `_ or -`ttk::widget `_ man +`options `_ or +`ttk::widget `_ man pages, while others are found in the man page for a specific widget class. -You'll also find that many Tkinter methods have compound names, e.g., -:func:`winfo_x`, :func:`winfo_height`, :func:`winfo_viewable`. You'd find -documentation for all of these in the -`winfo `_ man page. +You'll also find that many Tkinter methods have compound names, for example, +:meth:`~Misc.winfo_x`, :meth:`~Misc.winfo_height`, +:meth:`~Misc.winfo_viewable`. +You'd find documentation for all of these in the +`winfo `_ man page. .. note:: Somewhat confusingly, there are also methods on all Tkinter widgets @@ -517,14 +452,16 @@ from a thread other than the one that created the :class:`Tk` object, an event is posted to the interpreter's event queue, and when executed, the result is returned to the calling Python thread. -Tcl/Tk applications are normally event-driven, meaning that after initialization, -the interpreter runs an event loop (i.e. :func:`Tk.mainloop`) and responds to events. -Because it is single-threaded, event handlers must respond quickly, otherwise they -will block other events from being processed. To avoid this, any long-running -computations should not run in an event handler, but are either broken into smaller -pieces using timers, or run in another thread. This is different from many GUI -toolkits where the GUI runs in a completely separate thread from all application -code including event handlers. +Tcl/Tk applications are normally event-driven, meaning that after +initialization, the interpreter runs an event loop (that is, +:meth:`Tk.mainloop `) and responds to events. +Because it is single-threaded, event handlers must respond quickly, otherwise +they will block other events from being processed. +To avoid this, any long-running computations should not run in an event +handler, but are either broken into smaller pieces using timers, or run in +another thread. +This is different from many GUI toolkits where the GUI runs in a completely +separate thread from all application code including event handlers. If the Tcl interpreter is not running the event loop and processing events, any :mod:`!tkinter` calls made from threads other than the one running the Tcl @@ -532,7 +469,9 @@ interpreter will fail. A number of special cases exist: -* Tcl/Tk libraries can be built so they are not thread-aware. In this case, +* Tcl/Tk libraries built without thread support are now rare: Tcl/Tk 9.0 (the + bundled version) is always thread-aware, so this case only arises with some + older 8.x builds. When the library is not thread-aware, :mod:`!tkinter` calls the library from the originating Python thread, even if this is different than the thread that created the Tcl interpreter. A global lock ensures only one call occurs at a time. @@ -552,13 +491,13 @@ A number of special cases exist: called from the thread that created the Tcl interpreter. -Handy Reference +Handy reference --------------- .. _tkinter-setting-options: -Setting Options +Setting options ^^^^^^^^^^^^^^^ Options control things like the color and border width of a widget. Options can @@ -594,11 +533,11 @@ document. Some options don't apply to some kinds of widgets. Whether a given widget responds to a particular option depends on the class of the widget; buttons have a ``command`` option, labels do not. -The options supported by a given widget are listed in that widget's man page, or -can be queried at runtime by calling the :meth:`config` method without -arguments, or by calling the :meth:`keys` method on that widget. The return -value of these calls is a dictionary whose key is the name of the option as a -string (for example, ``'relief'``) and whose values are 5-tuples. +The options supported by a given widget are listed in that widget's man page, +or can be queried at runtime by calling the :meth:`~Misc.config` method without +arguments, or by calling the :meth:`~Misc.keys` method on that widget. +The return value of these calls is a dictionary whose key is the name of the +option as a string (for example, ``'relief'``) and whose values are 5-tuples. Some options, like ``bg`` are synonyms for common options with long names (``bg`` is shorthand for "background"). Passing the ``config()`` method the name @@ -630,7 +569,9 @@ Of course, the dictionary printed will include all the options available and their values. This is meant only as an example. -The Packer +.. _pack-the-packer: + +The packer ^^^^^^^^^^ .. index:: single: packing (widgets) @@ -650,10 +591,11 @@ Additionally, the arrangement is dynamically adjusted to accommodate incremental changes to the configuration, once it is packed. Note that widgets do not appear until they have had their geometry specified -with a geometry manager. It's a common early mistake to leave out the geometry -specification, and then be surprised when the widget is created but nothing -appears. A widget will appear only after it has had, for example, the packer's -:meth:`pack` method applied to it. +with a geometry manager. +It's a common early mistake to leave out the geometry specification, and then +be surprised when the widget is created but nothing appears. +A widget will appear only after it has had, for example, the packer's +:meth:`~Pack.pack` method applied to it. The pack() method can be called with keyword-option/value pairs that control where the widget is to appear within its container, and how it is to behave when @@ -664,7 +606,7 @@ the main application window is resized. Here are some examples:: fred.pack(expand=1) -Packer Options +Packer options ^^^^^^^^^^^^^^ For more extensive information on the packer and the options that it can take, @@ -674,7 +616,7 @@ anchor Anchor type. Denotes where the packer is to place each content in its parcel. expand - Boolean, ``0`` or ``1``. + boolean, ``0`` or ``1``. fill Legal values: ``'x'``, ``'y'``, ``'both'``, ``'none'``. @@ -689,7 +631,9 @@ side Legal values are: ``'left'``, ``'right'``, ``'top'``, ``'bottom'``. -Coupling Widget Variables +.. _coupling-widget-variables: + +Coupling widget variables ^^^^^^^^^^^^^^^^^^^^^^^^^ The current-value setting of some widgets (like text entry widgets) can be @@ -706,10 +650,11 @@ defined in :mod:`!tkinter`. There are many useful subclasses of Variable already defined: :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and -:class:`BooleanVar`. To read the current value of such a variable, call the -:meth:`get` method on it, and to change its value you call the :meth:`!set` -method. If you follow this protocol, the widget will always track the value of -the variable, with no further intervention on your part. +:class:`BooleanVar`. +To read the current value of such a variable, call the :meth:`~Variable.get` +method on it, and to change its value you call the :meth:`!set` method. +If you follow this protocol, the widget will always track the value of the +variable, with no further intervention on your part. For example:: @@ -743,7 +688,7 @@ For example:: myapp = App(root) myapp.mainloop() -The Window Manager +The window manager ^^^^^^^^^^^^^^^^^^ .. index:: single: window manager (widgets) @@ -756,11 +701,16 @@ subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods directly. To get at the toplevel window that contains a given widget, you can often just -refer to the widget's :attr:`master`. Of course if the widget has been packed inside of -a frame, the :attr:`!master` won't represent a toplevel window. To get at the toplevel -window that contains an arbitrary widget, you can call the :meth:`_root` method. -This method begins with an underscore to denote the fact that this function is -part of the implementation, and not an interface to Tk functionality. +refer to the widget's :attr:`~Tk.master`. +Of course if the widget has been packed inside of a frame, the :attr:`!master` +won't represent a toplevel window. +To get at the toplevel window that contains an arbitrary widget, you can call +the :meth:`~Misc.winfo_toplevel` method. +There is also a :meth:`!_root` method; it begins with an underscore to denote +the fact that this function is part of the implementation, and not an interface +to Tk functionality. +It returns the application's root window rather than the nearest enclosing +toplevel. Here are some examples of typical usage:: @@ -784,7 +734,7 @@ Here are some examples of typical usage:: myapp.mainloop() -Tk Option Data Types +Tk option data types ^^^^^^^^^^^^^^^^^^^^ .. index:: single: Tk Option Data Types @@ -794,10 +744,11 @@ anchor ``"s"``, ``"sw"``, ``"w"``, ``"nw"``, and also ``"center"``. bitmap - There are eight built-in, named bitmaps: ``'error'``, ``'gray25'``, - ``'gray50'``, ``'hourglass'``, ``'info'``, ``'questhead'``, ``'question'``, - ``'warning'``. To specify an X bitmap filename, give the full path to the file, - preceded with an ``@``, as in ``"@/usr/contrib/bitmap/gumby.bit"``. + There are ten built-in, named bitmaps: ``'error'``, ``'gray12'``, + ``'gray25'``, ``'gray50'``, ``'gray75'``, ``'hourglass'``, ``'info'``, + ``'questhead'``, ``'question'``, ``'warning'``. To specify an X bitmap + filename, give the full path to the file, preceded with an ``@``, as in + ``"@/usr/contrib/bitmap/gumby.bit"``. boolean You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"``. @@ -817,7 +768,7 @@ color cursor The standard X cursor names from :file:`cursorfont.h` can be used, without the - ``XC_`` prefix. For example to get a hand cursor (:const:`XC_hand2`), use the + ``XC_`` prefix. For example to get a hand cursor (``XC_hand2``), use the string ``"hand2"``. You can also specify a bitmap and mask file of your own. See page 179 of Ousterhout's book. @@ -829,9 +780,11 @@ distance as ``"3.5i"``. font - Tk uses a list font name format, such as ``{courier 10 bold}``. Font sizes with - positive numbers are measured in points; sizes with negative numbers are - measured in pixels. + Tk uses a font description such as ``{courier 10 bold}``; in + :mod:`!tkinter` this is most naturally passed as a tuple of + ``(family, size, *styles)`` (or as the equivalent string + ``"Courier 10 bold"``). Font sizes with positive numbers are measured in + points; sizes with negative numbers are measured in pixels. geometry This is a string of the form ``widthxheight``, where width and height are @@ -848,7 +801,8 @@ region relief Determines what the border style of a widget will be. Legal values are: - ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, and ``"ridge"``. + ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, ``"ridge"``, and + ``"solid"``. scrollcommand This is almost always the :meth:`!set` method of some scrollbar widget, but can @@ -859,7 +813,7 @@ wrap .. _Bindings-and-Events: -Bindings and Events +Bindings and events ^^^^^^^^^^^^^^^^^^^ .. index:: @@ -875,7 +829,10 @@ of the bind method is:: where: sequence - is a string that denotes the target kind of event. (See the + is a string that denotes the target kind of event. Physical events use the + ```` form (for example ``""`` or + ``""``); application-defined virtual events use double angle + brackets, as in ``"<>"``. (See the :manpage:`bind(3tk)` man page, and page 201 of John Ousterhout's book, :title-reference:`Tcl and the Tk Toolkit (2nd edition)`, for details). @@ -921,9 +878,13 @@ they are denoted in Tk, which can be useful when referring to the Tk man pages. +----+---------------------+----+---------------------+ | %y | y | %Y | y_root | +----+---------------------+----+---------------------+ +| %# | serial | %b | num | ++----+---------------------+----+---------------------+ +| %d | detail | %D | delta | ++----+---------------------+----+---------------------+ -The index Parameter +The index parameter ^^^^^^^^^^^^^^^^^^^ A number of widgets require "index" parameters to be passed. These are used to @@ -988,7 +949,7 @@ option (other options are available as well). :meth:`!copy`. The image object can then be used wherever an ``image`` option is supported by -some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a +some widget (for example, labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is deleted, the image data is deleted as well, and Tk will display an empty box wherever the image was used. @@ -998,51 +959,5812 @@ wherever the image was used. The `Pillow `_ package adds support for formats such as BMP, JPEG, TIFF, and WebP, among others. -.. _tkinter-file-handlers: -File Handlers -------------- +Reference +--------- -Tk allows you to register and unregister a callback function which will be -called from the Tk mainloop when I/O is possible on a file descriptor. -Only one handler may be registered per file descriptor. Example code:: +.. currentmodule:: tkinter - import tkinter - widget = tkinter.Tk() - mask = tkinter.READABLE | tkinter.WRITABLE - widget.tk.createfilehandler(file, mask, callback) - ... - widget.tk.deletefilehandler(file) +This section documents the classes, methods, functions and constants of the +:mod:`!tkinter` module. +Most of them wrap Tcl/Tk commands; consult the official Tcl/Tk manual pages for +the full list of widget options and further details. -This feature is not available on Windows. +.. exception:: TclError -Since you don't know how many bytes are available for reading, you may not -want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` -:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, -since these will insist on reading a predefined number of bytes. -For sockets, the :meth:`~socket.socket.recv` or -:meth:`~socket.socket.recvfrom` methods will work fine; for other files, -use raw reads or ``os.read(file.fileno(), maxbytecount)``. + The exception raised when a call into the Tcl interpreter fails, for example + when a widget is given an unknown option or an invalid value. +Base and mixin classes +^^^^^^^^^^^^^^^^^^^^^^ -.. method:: Widget.tk.createfilehandler(file, mask, func) +.. class:: Misc() - Registers the file handler callback function *func*. The *file* argument - may either be an object with a :meth:`~io.IOBase.fileno` method (such as - a file or socket object), or an integer file descriptor. The *mask* - argument is an ORed combination of any of the three constants below. - The callback is called as follows:: + The :class:`!Misc` class is a mix-in inherited by :class:`Tk` and, through + :class:`BaseWidget`, by every widget. + It provides the large set of methods common to all Tk objects: querying + window information, managing event bindings and the event loop, controlling + the keyboard focus and pointer grabs, accessing the selection, clipboard and + option database, and assorted utility and introspection services. + Because they are inherited, these methods are available on every widget and + on the :class:`Tk` application object, and are documented here once rather + than repeated for each widget. - callback(file, mask) + .. method:: cget(key) + Return the current value of the configuration option named *key* for this + widget, as a string. + The expression ``widget[key]`` is equivalent and may be used instead. -.. method:: Widget.tk.deletefilehandler(file) + .. method:: config(cnf=None, **kw) + :no-typesetting: - Unregisters a file handler. + .. method:: configure(cnf=None, **kw) + Query or modify the configuration options of the widget. + With no arguments, return a dictionary mapping every available option + name to a tuple describing it (its name, X resource name, X resource + class, default value and current value). + If a single option name is given as a string, return the tuple for just + that option. + If one or more keyword arguments are given, or a dictionary is passed as + *cnf*, set each named option to the corresponding value; the expression + ``widget[key] = value`` sets a single option in the same way. -.. data:: READABLE - WRITABLE - EXCEPTION + :meth:`config` is an alias of :meth:`!configure`. - Constants used in the *mask* arguments. + .. method:: keys() + + Return a list of the names of all configuration options of this widget. + + .. method:: getboolean(s) + + Interpret the string *s* as a Tcl boolean and return the corresponding + :class:`bool`. + Tcl accepts values such as ``'1'``, ``'0'``, ``'yes'``, ``'no'``, + ``'true'`` and ``'false'``. + Raise :exc:`ValueError` if *s* is not a valid boolean. + + .. method:: getdouble(s) + + Interpret the string *s* as a Tcl floating-point number and return it as + a :class:`float`. + Raise :exc:`ValueError` if *s* is not a valid number. + + .. versionadded:: 3.5 + + + .. method:: getint(s) + + Interpret the string *s* as a Tcl integer and return it as an + :class:`int`. + Raise :exc:`ValueError` if *s* is not a valid integer. + + .. method:: getvar(name='PY_VAR') + + Return the value of the Tcl global variable named *name*. + + .. method:: setvar(name='PY_VAR', value='1') + + Set the Tcl global variable named *name* to *value*. + + The :meth:`!getvar` and :meth:`!setvar` methods give direct access to Tcl + variables. + In most code you will instead use a :class:`Variable` subclass such as + :class:`StringVar` or :class:`IntVar`, which wraps a Tcl variable and + converts its value to and from a Python type. + + .. method:: register(func, subst=None, needcleanup=1) + + Register the Python callable *func* as a Tcl command and return the name + of the new command as a string. + Whenever Tcl invokes that command, *func* is called; if *subst* is given, + it is applied to the command's arguments first. + This is the mechanism used internally to turn Python callbacks into the + command names passed to Tk options such as *command*. + Unless *needcleanup* is false, the command is deleted automatically when + the widget is destroyed. + + .. versionchanged:: 3.13 + The arguments passed to *func* are no longer converted to strings. + + .. method:: deletecommand(name) + + Delete the Tcl command named *name*, such as one previously returned by + :meth:`register`. + + .. method:: nametowidget(name) + + Return the widget instance corresponding to the Tk pathname *name*. + + .. method:: send(interp, cmd, *args) + + Send the Tcl command *cmd*, with the given *args*, to the Tcl interpreter + registered under the name *interp*, and return its result. + This is not available on all platforms. + + .. method:: destroy() + + Destroy this widget and all of its descendant widgets, and delete the Tcl + commands associated with them. + + .. method:: lift(aboveThis=None) + :no-typesetting: + + .. method:: tkraise(aboveThis=None) + + Raise this widget in the stacking order so that it is drawn on top of its + siblings. + If *aboveThis* is given, the widget is moved to be just above it in the + stacking order instead. + + :meth:`lift` is an alias of :meth:`!tkraise`. + + .. method:: lower(belowThis=None) + + Lower this widget in the stacking order so that it is drawn beneath its + siblings. + If *belowThis* is given, the widget is moved to be just below it in the + stacking order instead. + + :meth:`tkraise`/:meth:`lift` and :meth:`lower` are overridden by the + :class:`Canvas` widget, + where they restack canvas items instead. + + .. method:: image_names() + + Return the names of all images that currently exist in the Tcl + interpreter. + + This is overridden by the :class:`Text` widget, + where :meth:`!image_names` returns the names of its embedded images + instead. + + .. method:: image_types() + + Return the available image types, such as ``'photo'`` and ``'bitmap'``. + + .. method:: anchor(anchor=None) + :no-typesetting: + + .. method:: grid_anchor(anchor=None) + + Set the anchor that controls where the grid is placed inside this + container when the container is larger than the grid and no row or column + has a non-zero weight. + *anchor* is one of the usual anchor strings, such as ``'nw'`` (the + default) or ``'center'``. + Called with no argument, this method has no effect. + + :meth:`anchor` is an alias of :meth:`!grid_anchor`. + + .. versionadded:: 3.3 + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Return the bounding box, in pixels, of a region of the grid laid out in + this container, as a 4-tuple ``(xoffset, yoffset, width, height)``. + With no arguments the bounding box of the whole grid is returned. + If *column* and *row* are given, the box spans from the cell at row and + column 0 to that cell; if *col2* and *row2* are also given, it spans from + the cell (*column*, *row*) to the cell (*col2*, *row2*). + + :meth:`bbox` is an alias of :meth:`!grid_bbox`, + except on :class:`Canvas`, :class:`Listbox`, :class:`Spinbox`, + :class:`Text`, :class:`ttk.Entry ` and + :class:`ttk.Treeview `, + which provide their own :meth:`!bbox` method. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Query or set the properties of the column (or columns) *index* of the + grid managed by this container. + *index* may be a column number; when setting options it may also be a + list of column numbers, the string ``'all'`` to affect every column, or + a child widget whose occupied columns are affected. + The supported options are: + + *minsize* + The column's minimum size, in pixels. + + *weight* + An integer setting how much of any extra space is apportioned to the + column. + A weight of ``0`` keeps the column at its requested size, and a column + of weight two grows twice as fast as a column of weight one. + + *uniform* + The name of a uniform group. + Columns sharing a non-empty group name are kept in sizes that are + strictly proportional to their weights. + + *pad* + Extra space, in pixels, added to the largest widget in the column when + computing the column's size. + + With a single option name, return that option's value; with no options, + return a dictionary of all of them. + + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Query or set the properties of the row (or rows) *index* of the grid + managed by this container. + *index* is interpreted as for :meth:`grid_columnconfigure`, and the + supported options (*minsize*, *weight*, *uniform* and *pad*) are the + same, applied to a row instead of a column. + + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: grid_location(x, y) + + Return the ``(column, row)`` of the grid cell that contains the pixel at + position (*x*, *y*), given in pixels relative to this container. + For locations above or to the left of the grid, ``-1`` is returned for + the corresponding coordinate. + + .. method:: grid_propagate() + grid_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the grid geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Return the size of the grid managed by this container as a + ``(columns, rows)`` tuple. + + :meth:`size` is an alias of :meth:`!grid_size`, + except on :class:`Listbox` and + :class:`ttk.Treeview `, + which provide their own :meth:`!size` method. + + .. method:: grid_slaves(row=None, column=None) + + Return a list of the child widgets managed in this container's grid, most + recently managed first. + If *row* or *column* is given, only the children in that row or column + are returned. + + .. method:: grid_content(row=None, column=None) + + Same as :meth:`grid_slaves`: return the child widgets managed in this + container's grid, optionally restricted to a given *row* or *column*. + + .. versionadded:: 3.15 + + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the pack geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Return a list of the child widgets managed by this container with the + pack geometry manager, in packing order. + + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: pack_content() + + Same as :meth:`pack_slaves`: return the child widgets managed by this + container with the pack geometry manager, in packing order. + + :meth:`content` is an alias of :meth:`!pack_content`. + + .. versionadded:: 3.15 + + + .. method:: place_slaves() + + Return a list of the child widgets managed by this container with the + place geometry manager. + + .. method:: place_content() + + Same as :meth:`place_slaves`: return the child widgets managed by this + container with the place geometry manager. + + .. versionadded:: 3.15 + + The methods with the ``bind`` and ``unbind`` prefixes associate event + patterns with callbacks and remove those associations. + + .. method:: bind(sequence=None, func=None, add=None) + + Bind the event pattern *sequence* on this widget to the callable *func*. + + *sequence* is an event pattern, such as ``''`` (a mouse click) + or ``''``, optionally a concatenation of several such + patterns that must occur shortly after one another. + When the event occurs, *func* is called with an :class:`Event` instance + describing it as its only argument; if *func* returns the string + ``'break'``, no further bindings for the event are invoked. + + If *add* is true, *func* is added to any functions already bound to + *sequence*; otherwise it replaces them. + The binding applies only to this widget. + + :meth:`!bind` returns a string identifier (a *funcid*) that can later be + passed to :meth:`unbind` to remove the binding without leaking the + associated Tcl command. + + If *func* is omitted, return the binding currently associated with + *sequence*; if *sequence* is also omitted, return a list of all the + sequences for which bindings exist on this widget. + + .. method:: bind_class(className, sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the binding tag *className* rather + than to a single widget, so that the binding applies to every widget + having that tag. + *className* is usually the name of a widget class, such as ``'Button'``, + in which case the binding affects all widgets of that class. + The set of binding tags for a widget can be inspected and changed with + :meth:`bindtags`. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: bind_all(sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the special binding tag ``'all'``, + so that the binding applies to every widget in the application. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: unbind(sequence, funcid=None) + + Remove bindings for the event pattern *sequence* on this widget. + + If *funcid* is given, only the function identified by it (a value + returned from a previous call to :meth:`bind`) is removed, and its + associated Tcl command is deleted. + Otherwise all bindings for *sequence* are destroyed, leaving it unbound. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound; other callbacks + bound to *sequence* are kept. + + + .. method:: unbind_class(className, sequence) + + Remove all bindings for the event pattern *sequence* from the binding tag + *className*. + See :meth:`bind_class`. + + .. method:: unbind_all(sequence) + + Remove all bindings for the event pattern *sequence* from the special + binding tag ``'all'``. + See :meth:`bind_all`. + + .. method:: bindtags(tagList=None) + + If *tagList* is omitted, return a tuple of the binding tags associated + with this widget. + When an event occurs in a widget, it is applied to each of the widget's + binding tags in order, and for each tag the most specific matching + binding is executed. + By default a widget has four binding tags: its own pathname, its widget + class, the pathname of its nearest toplevel ancestor, and ``'all'``, in + that order. + + If *tagList* is given, it must be a sequence of strings; the widget's + binding tags are set to its elements, which determines the order in which + bindings are evaluated. + + The methods with the ``event_`` prefix define virtual events and generate + events programmatically. + + .. method:: event_add(virtual, *sequences) + + Associate the virtual event *virtual*, whose name has the form + ``'<>'``, with each of the physical event patterns given by + *sequences*, so that the virtual event triggers whenever any of them + occurs. + If *virtual* is already defined, the new sequences are added to its + existing ones. + + .. method:: event_delete(virtual, *sequences) + + Remove each of *sequences* from those associated with the virtual event + *virtual*. + Sequences that are not currently associated with *virtual* are ignored. + If no *sequences* are given, all physical event sequences are removed, so + that *virtual* no longer triggers. + + .. method:: event_generate(sequence, **kw) + + Generate the event *sequence* on this widget and arrange for it to be + processed just as if it had come from the window system. + *sequence* must be a single event pattern, such as ``''`` or + ``'<>'``, not a concatenation of several. + Keyword arguments specify additional fields of the event, for example *x* + and *y* for the pointer position, or *when* to control when the event is + processed; refer to the Tk ``event`` manual page for the full list. + + .. method:: event_info(virtual=None) + + If *virtual* is omitted, return a tuple of all the virtual events that + are currently defined. + If *virtual* is given, return a tuple of the physical event sequences + currently associated with it, or an empty tuple if it is not defined. + + The methods with the ``after`` prefix schedule callbacks to run after a + delay or when the application is idle. + + .. method:: after(ms, func=None, *args, **kw) + + Schedule the callable *func* to be called after *ms* milliseconds, with + *args* and *kw* passed to it as positional and keyword arguments. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + If *func* is omitted, sleep for *ms* milliseconds instead, processing no + events during that time, and return ``None``. + + .. versionchanged:: 3.10 + *func* can now be any callable object, not only a function. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_cancel(id) + + Cancel a callback previously scheduled with :meth:`after` or + :meth:`after_idle`. + *id* must be an identifier returned by one of those methods; passing a + value that is not such an identifier raises :exc:`ValueError`. + If the callback has already run or been cancelled, this has no effect. + + .. versionchanged:: 3.7 + Passing ``None`` (or any false value) as *id* now raises + :exc:`ValueError`. + + + .. method:: after_idle(func, *args, **kw) + + Schedule the callable *func* to be called, with *args* and *kw* passed to + it, when the Tk main loop next becomes idle, that is, when it has no + other events to process. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_info(id=None) + + If *id* is omitted, return a tuple of the identifiers of all callbacks + currently scheduled with :meth:`after` and :meth:`after_idle` for this + interpreter. + + If *id* is given, it must identify a callback that has not yet run or + been cancelled, and the return value is a tuple ``(script, type)``, where + *script* refers to the function to be called and *type* is either + ``'idle'`` or ``'timer'``. + A :exc:`TclError` is raised if *id* does not exist. + + .. versionadded:: 3.13 + + + .. method:: mainloop(n=0) + + Enter the Tk event loop, which processes events until all windows are + destroyed. + This is normally called once, on the root window, to run the application. + + .. method:: quit() + + Quit the Tcl interpreter, causing :meth:`mainloop` to return. + + .. method:: update() + + Enter the event loop until all pending events, including idle callbacks, + have been processed. + This brings the display up to date and handles any events that are + already queued, then returns. + + .. method:: update_idletasks() + + Enter the event loop until all pending idle callbacks have been called. + This updates the display of windows, for example after geometry changes, + but does not process events caused by the user. + + .. method:: waitvar(name='PY_VAR') + :no-typesetting: + + .. method:: wait_variable(name='PY_VAR') + + Wait until the Tcl variable *name* is modified, continuing to process + events in the meantime so that the application stays responsive. + *name* is usually a :class:`Variable` instance, such as an + :class:`IntVar` or :class:`StringVar`. + + :meth:`waitvar` is an alias of :meth:`!wait_variable`. + + .. method:: wait_window(window=None) + + Wait until *window* is destroyed, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for the user to finish interacting with a + dialog box. + + .. method:: wait_visibility(window=None) + + Wait until the visibility state of *window* changes, for example when it + first appears on the screen, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for a newly created window to become + visible before acting on it. + + The methods with the ``focus_`` prefix manage the keyboard focus. + + .. method:: focus_set() + :no-typesetting: + + .. method:: focus() + + Direct the keyboard input focus for this widget's display to this widget. + If the application does not currently have the input focus on this + widget's display, the widget is remembered as the focus window for its + top level, and the focus will be redirected to it the next time the + window manager gives the focus to the top level. + :meth:`focus` is an alias of :meth:`!focus_set`, + except on the :class:`Canvas` and + :class:`ttk.Treeview ` widgets, + which provide their own :meth:`!focus` method. + + .. method:: focus_force() + + Direct the keyboard input focus to this widget even if the application + does not currently have the input focus for the widget's display. + This method should be used sparingly, if at all; normally an application + should wait for the window manager to give it the focus rather than + claiming it. + + .. method:: focus_get() + + Return the widget that currently has the keyboard focus in the + application, or ``None`` if no widget in the application has the focus. + Use :meth:`focus_displayof` to work correctly with several displays. + + .. method:: focus_displayof() + + Return the widget that currently has the keyboard focus on the display + where this widget is located, or ``None`` if no widget in the application + has the focus on that display. + + .. method:: focus_lastfor() + + Return the most recent widget to have had the keyboard focus among all + the widgets in the same top level as this widget; this is the widget that + will receive the focus the next time the window manager gives the focus + to the top level. + If no widget in that top level has ever had the focus, or if the most + recent focus widget has been deleted, the top level itself is returned. + + .. method:: tk_focusFollowsMouse() + + Reconfigure Tk to use an implicit focus model in which the focus is set + to a widget whenever the mouse pointer enters it. + This cannot easily be disabled once enabled. + + .. method:: tk_focusNext() + + Return the next widget after this one in the keyboard traversal order, or + ``None`` if there is none. + The traversal order goes first to the next child, then recursively to the + children of that child, and then to the next sibling higher in the + stacking order. + A widget is skipped if its ``takefocus`` option is set to ``0``. + This method is used in the default bindings for the :kbd:`Tab` key. + + .. method:: tk_focusPrev() + + Return the previous widget before this one in the keyboard traversal + order, or ``None`` if there is none. + See :meth:`tk_focusNext` for how the order is defined. + This method is used in the default bindings for the :kbd:`Shift-Tab` key. + + The methods with the ``grab_`` prefix set and query the input grab, which + directs all input events to a single widget. + + .. method:: grab_set() + + Set a local grab on this widget. + A grab confines pointer events to this widget and its descendants: while + the pointer is outside the widget's subtree, button presses and releases + and pointer motion are reported to the grab widget, and windows outside + the subtree become insensitive until the grab is released. + A local grab affects only the grabbing application. + Any grab previously set by this application on the widget's display is + automatically released. + Setting a grab is the usual way to make a dialog modal: while the grab is + in effect the user cannot interact with the other windows of the + application. + + .. method:: grab_set_global() + + Set a global grab on this widget. + A global grab is like the local grab set by :meth:`grab_set`, but it + locks out all other applications on the screen, so that only this + widget's subtree is sensitive to pointer events, and it also grabs the + keyboard. + Use with caution: it is easy to render a display unusable with a global + grab, since other applications stop receiving events until it is + released. + + .. method:: grab_release() + + Release the grab on this widget if there is one; otherwise do nothing. + + .. method:: grab_current() + + Return the widget that currently holds the grab in this application for + this widget's display, or ``None`` if there is no such widget. + + .. method:: grab_status() + + Return ``None`` if no grab is currently set on this widget, ``"local"`` + if a local grab is set, or ``"global"`` if a global grab is set. + + The methods with the ``selection_`` prefix retrieve and manage the X + selection. + + .. method:: selection_clear(**kw) + + Clear the X selection, so that no window owns it anymore. + The selection to clear is given by the keyword argument *selection*, an + atom name such as ``'PRIMARY'`` or ``'CLIPBOARD'``; it defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display on which to operate, and defaults to this widget. + + This is overridden by the :class:`Entry`, :class:`Listbox` and + :class:`Spinbox` widgets, + where :meth:`!selection_clear` clears the widget's own selection instead. + + .. method:: selection_get(**kw) + + Return the contents of the current X selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The keyword argument *type* specifies the form in which the data is to be + returned (the desired conversion target), an atom name such as + ``'STRING'`` or ``'FILE_NAME'``; it defaults to ``STRING``, except on + X11, where ``UTF8_STRING`` is tried first and ``STRING`` is used as a + fallback. + The *displayof* keyword argument names a widget that determines the + display from which to retrieve the selection, and defaults to this + widget. + + .. method:: selection_handle(command, **kw) + + Register *command* as a handler to supply the X selection owned by this + widget when another application requests it. + When the selection is retrieved, *command* is called with two arguments, + the starting character offset and the maximum number of characters to + return, and must return at most that many characters of the selection + starting at that offset; for very long selections it is called repeatedly + with increasing offsets. + The keyword argument *selection* names the selection (default + ``PRIMARY``) and the keyword argument *type* gives the form of the + selection that the handler supplies (such as ``'STRING'`` or + ``'FILE_NAME'``, default ``STRING``). + + .. method:: selection_own(**kw) + + Make this widget the owner of the X selection on its display. + The previous owner, if any, is notified that it has lost the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + + .. method:: selection_own_get(**kw) + + Return the widget in this application that owns the X selection on the + display containing this widget, or ``None`` if no widget in this + application owns the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display to query, and defaults to this widget. + + The methods with the ``clipboard_`` prefix manage the clipboard. + + .. method:: clipboard_append(string, **kw) + + Append *string* to the Tk clipboard and claim ownership of the clipboard + on this widget's display. + Before appending, the clipboard should be emptied with + :meth:`clipboard_clear`; all appends should be completed before returning + to the event loop so that the clipboard is updated atomically. + The keyword argument *type* specifies the form of the data, an atom name + such as ``'STRING'`` or ``'FILE_NAME'`` (default ``STRING``), and the + keyword argument *format* specifies the representation used to transmit + it (default ``STRING``). + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + The contents can be retrieved with :meth:`clipboard_get` or + :meth:`selection_get`. + + .. method:: clipboard_clear(**kw) + + Claim ownership of the clipboard on this widget's display and remove any + previous contents. + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + + .. method:: clipboard_get(**kw) + + Retrieve data from the clipboard on this widget's display. + The keyword argument *type* specifies the form in which the data is to be + returned, an atom name such as ``'STRING'`` or ``'FILE_NAME'``; it + defaults to ``STRING``, except on X11, where ``UTF8_STRING`` is tried + first and ``STRING`` is used as a fallback. + The *displayof* keyword argument names a widget that determines the + display, and defaults to the root window of the application. + This is equivalent to ``selection_get(selection='CLIPBOARD')``. + + The methods with the ``option_`` prefix query and modify the Tk option + database. + + .. method:: option_add(pattern, value, priority=None) + + Add an option to the Tk option database that associates *value* with + *pattern*. + *pattern* consists of names and/or classes separated by asterisks or + dots, in the usual X format. + *priority* is an integer between 0 and 100, or one of the symbolic names + ``'widgetDefault'`` (20), ``'startupFile'`` (40), ``'userDefault'`` (60), + or ``'interactive'`` (80); it defaults to ``interactive``. + + .. method:: option_clear() + + Clear the Tk option database. + Default options from the :envvar:`!RESOURCE_MANAGER` property or the + :file:`.Xdefaults` file are reloaded automatically the next time an + option is added to or removed from the database. + + .. method:: option_get(name, className) + + Return the value of the option matching this widget under *name* and + *className* from the Tk option database, or an empty string if there is + no matching entry. + When several entries match, the one with the highest priority is + returned, and among entries of equal priority the most recently added + one. + + .. method:: option_readfile(fileName, priority=None) + + Read the file named *fileName*, which should have the standard format for + an X resource database such as :file:`.Xdefaults`, and add all the + options it specifies to the Tk option database. + *priority* is interpreted as for :meth:`option_add` and defaults to + ``interactive``. + + .. method:: bell(displayof=0) + + Ring the bell on the display for this widget, using the display's current + bell-related settings, and reset the screen saver for the screen. + If *displayof* is given as a widget, the bell is rung on that widget's + display instead. + + .. method:: tk_setPalette(background, /) + tk_setPalette(*args, **kw) + + Set a new color scheme for all Tk widget elements. + Existing widgets are updated and the option database is changed so that + future widgets use the new colors. + A single color argument is taken as the normal background color, from + which a complete palette is computed. + Alternatively, the arguments may be given as keyword *name*/*value* pairs + naming individual options in the option database. + The recognized option names are ``activeBackground``, + ``activeForeground``, ``background``, ``disabledForeground``, + ``foreground``, ``highlightBackground``, ``highlightColor``, + ``insertBackground``, ``selectColor``, ``selectBackground``, + ``selectForeground``, and ``troughColor``; reasonable defaults are + computed for any that are not specified. + + .. method:: tk_bisque() + + Restore the application's colors to the light brown (bisque) color scheme + used in Tk 3.6 and earlier versions. + Provided for backward compatibility. + + .. method:: tk_strictMotif(boolean=None) + + Query or set whether Tk's look and feel should strictly adhere to Motif. + A true *boolean* value enables strict Motif compliance (for example, no + color change when the mouse passes over a slider). + Return the resulting setting. + + .. method:: tk_appname(name=None) + + Query or set the name used to communicate with this application through + the ``send`` command. + With no argument, return the current name; otherwise change it to *name* + and return the actual name set, which may have a suffix appended to keep + it unique among the applications on the display. + + .. versionadded:: next + + .. method:: tk_useinputmethods(boolean=None, *, displayof=0) + + Query or set whether Tk uses the X Input Methods (XIM) for filtering + events, and return the resulting state. + This is significant only on X11; if XIM support is not available it + always returns ``False``. + + .. versionadded:: next + + .. method:: tk_caret(*, x=None, y=None, height=None) + + Set or query the caret location for the widget's display. + The caret is the per-display insertion position used for global focus + indication (for accessibility) and for placing the input method + (XIM or IME) window. + With no argument, return the current location as a dictionary with the + keys ``'x'``, ``'y'`` and ``'height'``; otherwise update the given + coordinates. + + .. versionadded:: next + + .. method:: tk_scaling(number=None, *, displayof=0) + + Query or set the scaling factor used by Tk to convert between physical + units (such as points, inches or millimeters) and pixels, expressed as + the number of pixels per point (where a point is 1/72 inch). + With no argument, return the current factor; otherwise set it to the + floating-point *number*. + + .. versionadded:: next + + .. method:: tk_inactive(reset=False, *, displayof=0) + + Return the number of milliseconds since the last time the user interacted + with the system, or ``-1`` if the windowing system does not support this. + If *reset* is true, reset the inactivity timer to zero instead and return + ``None``. + + .. versionadded:: next + + The methods with the ``busy_`` prefix manage the busy state of a window, + which shows a busy cursor and ignores user input. + + .. method:: busy(**kw) + :no-typesetting: + + .. method:: busy_hold(**kw) + :no-typesetting: + + .. method:: tk_busy(**kw) + :no-typesetting: + + .. method:: tk_busy_hold(**kw) + + Make this widget appear busy. + A transparent window is placed in front of the widget, so that it and all + of its descendants in the widget hierarchy are blocked from pointer + events and display a busy cursor. + Normally :meth:`update` should be called immediately afterwards to ensure + that the hold operation is in effect before the application starts its + processing. + + The only supported configuration option is *cursor*, the cursor to be + displayed while the widget is busy; it may have any of the values + accepted by :meth:`!configure`. + + :meth:`busy_hold`, :meth:`busy` and :meth:`tk_busy` are aliases of + :meth:`!tk_busy_hold`. + + .. versionadded:: 3.13 + + + .. method:: busy_configure(cnf=None, **kw) + :no-typesetting: + + .. method:: busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_configure(cnf=None, **kw) + + Query or modify the configuration options of the busy window. + The widget must have been previously made busy by :meth:`tk_busy_hold`. + With no arguments, return a dictionary describing all of the available + options; if *cnf* is the name of an option, return a tuple describing + that one option. + Otherwise set the given options to the given values. + Options may have any of the values accepted by :meth:`tk_busy_hold`. + + The option database is referenced through the widget name or class. + For example, if a :class:`Frame` widget named ``frame`` is to be made + busy, the busy cursor can be specified for it by either of the calls:: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + + :meth:`busy_configure`, :meth:`busy_config` and :meth:`tk_busy_config` + are aliases of :meth:`!tk_busy_configure`. + + .. versionadded:: 3.13 + + + .. method:: busy_cget(option) + :no-typesetting: + + .. method:: tk_busy_cget(option) + + Return the current value of the busy configuration *option*. + The widget must have been previously made busy by :meth:`tk_busy_hold`, + and *option* may have any of the values accepted by that method. + + :meth:`busy_cget` is an alias of :meth:`!tk_busy_cget`. + + .. versionadded:: 3.13 + + + .. method:: busy_forget() + :no-typesetting: + + .. method:: tk_busy_forget() + + Make this widget no longer busy, releasing the resources (including the + transparent window) allocated when it was made busy. + User events will again be received by the widget. + These resources are also released when the widget is destroyed. + + :meth:`busy_forget` is an alias of :meth:`!tk_busy_forget`. + + .. versionadded:: 3.13 + + + .. method:: busy_status() + :no-typesetting: + + .. method:: tk_busy_status() + + Return ``True`` if the widget is currently busy, ``False`` otherwise. + + :meth:`busy_status` is an alias of :meth:`!tk_busy_status`. + + .. versionadded:: 3.13 + + + .. method:: busy_current(pattern=None) + :no-typesetting: + + .. method:: tk_busy_current(pattern=None) + + Return a list of widgets that are currently busy. + If *pattern* is given, only busy widgets whose path names match the + pattern are returned. + + :meth:`busy_current` is an alias of :meth:`!tk_busy_current`. + + .. versionadded:: 3.13 + + The methods with the ``winfo_`` prefix retrieve information about windows + managed by Tk. + + .. method:: winfo_atom(name, displayof=0) + + Return the integer identifier for the atom whose name is *name*, creating + a new atom if none exists. + If *displayof* is given, the atom is looked up on the display of that + window; otherwise it is looked up on the display of the application's + main window. + + .. method:: winfo_atomname(id, displayof=0) + + Return the textual name for the atom whose integer identifier is *id*. + This is the inverse of :meth:`winfo_atom`. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_cells() + + Return the number of cells in the colormap for the widget. + + .. method:: winfo_children() + + Return a list containing the widgets that are children of the widget, in + stacking order from lowest to highest. + Toplevel windows are returned as children of their logical parents. + + .. method:: winfo_class() + + Return the class name of the widget. + + .. method:: winfo_colormapfull() + + Return ``True`` if the colormap for the widget is known to be full, + ``False`` otherwise. + + .. method:: winfo_containing(rootX, rootY, displayof=0) + + Return the widget containing the point given by *rootX* and *rootY*, or + ``None`` if no window in this application contains the point. + The coordinates are in screen units in the coordinate system of the root + window. + If *displayof* is given, the coordinates refer to the screen containing + that window; otherwise they refer to the screen of the application's main + window. + + .. method:: winfo_depth() + + Return the depth of the widget, that is, the number of bits per pixel. + + .. method:: winfo_exists() + + Return ``1`` if the widget exists, ``0`` otherwise. + + .. method:: winfo_fpixels(number) + + Return a floating-point value giving the number of pixels in the widget + corresponding to the screen distance *number* (for example, + ``"2.0c"`` or ``"1i"``). + The result may be fractional; for a rounded integer value use + :meth:`winfo_pixels`. + + .. method:: winfo_geometry() + + Return the geometry of the widget, in the form ``widthxheight+x+y``. + All dimensions are in pixels. + + .. method:: winfo_height() + + Return the height of the widget in pixels. + When a window is first created its height is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqheight`. + + .. method:: winfo_id() + + Return a low-level platform-specific identifier for the widget. + On Unix this is the X window identifier, and on Windows it is the window + handle. + + .. method:: winfo_interps(displayof=0) + + Return a tuple of the names of all Tcl interpreters currently registered + for a particular display. + If *displayof* is given, the return value refers to the display of that + window; otherwise it refers to the display of the application's main + window. + + .. method:: winfo_isdark() + + On macOS and Windows, return ``True`` if the widget is in "dark mode", + and ``False`` otherwise. + Always return ``False`` on X11. + + .. versionadded:: next + + Requires Tk 9.1 or newer. + + .. method:: winfo_ismapped() + + Return ``1`` if the widget is currently mapped, ``0`` otherwise. + + .. method:: winfo_manager() + + Return the name of the geometry manager currently responsible for the + widget, or an empty string if it is not managed by any geometry manager. + + .. method:: winfo_name() + + Return the widget's name within its parent, as opposed to its full path + name. + + .. method:: winfo_parent() + + Return the path name of the widget's parent, or an empty string if the + widget is the main window of the application. + + .. method:: winfo_pathname(id, displayof=0) + + Return the path name of the window whose identifier is *id*. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_pixels(number) + + Return the number of pixels in the widget corresponding to the screen + distance *number* (for example, ``"2.0c"`` or ``"1i"``). + The result is rounded to the nearest integer; for a fractional result use + :meth:`winfo_fpixels`. + + .. method:: winfo_pointerx() + + Return the pointer's *x* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_pointerxy() + + Return the pointer's coordinates as an ``(x, y)`` tuple, in pixels, + relative to the screen's root window (or virtual root, if one is in use). + Both coordinates are ``-1`` if the pointer is not on the same screen as + the widget. + + .. method:: winfo_pointery() + + Return the pointer's *y* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_reqheight() + + Return the widget's requested height in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_reqwidth() + + Return the widget's requested width in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_rgb(color) + + Return an ``(r, g, b)`` tuple of the red, green, and blue intensities, in + the range 0 to 65535, that correspond to *color* in the widget. + *color* may be specified in any of the forms acceptable for a color + option. + + .. method:: winfo_rootx() + + Return the *x* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_rooty() + + Return the *y* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_screen() + + Return the name of the screen associated with the widget, in the form + ``displayName.screenIndex``. + + .. method:: winfo_screencells() + + Return the number of cells in the default colormap for the widget's + screen. + + .. method:: winfo_screendepth() + + Return the depth of the root window of the widget's screen, that is, the + number of bits per pixel. + + .. method:: winfo_screenheight() + + Return the height of the widget's screen in pixels. + + .. method:: winfo_screenmmheight() + + Return the height of the widget's screen in millimeters. + + .. method:: winfo_screenmmwidth() + + Return the width of the widget's screen in millimeters. + + .. method:: winfo_screenvisual() + + Return the default visual class for the widget's screen, one of + ``"directcolor"``, ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, + ``"staticgray"``, or ``"truecolor"``. + + .. method:: winfo_screenwidth() + + Return the width of the widget's screen in pixels. + + .. method:: winfo_server() + + Return a string containing information about the server for the widget's + display. + The exact format of this string may vary from platform to platform. + + .. method:: winfo_toplevel() + + Return the top-of-hierarchy window containing the widget. + In standard Tk this is always a :class:`Toplevel` widget. + + .. method:: winfo_viewable() + + Return ``1`` if the widget and all of its ancestors up through the + nearest toplevel window are mapped, ``0`` otherwise. + + .. method:: winfo_visual() + + Return the visual class for the widget, one of ``"directcolor"``, + ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, ``"staticgray"``, + or ``"truecolor"``. + + .. method:: winfo_visualid() + + Return the X identifier for the visual for the widget. + + .. method:: winfo_visualsavailable(includeids=False) + + Return a list describing the visuals available for the widget's screen. + Each item consists of a visual class (see :meth:`winfo_visual`) followed + by an integer depth. + If *includeids* is true, the X identifier for the visual is also + included. + + .. method:: winfo_vrootheight() + + Return the height of the virtual root window associated with the widget + if there is one; otherwise return the height of the widget's screen. + + .. method:: winfo_vrootwidth() + + Return the width of the virtual root window associated with the widget if + there is one; otherwise return the width of the widget's screen. + + .. method:: winfo_vrootx() + + Return the *x* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_vrooty() + + Return the *y* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_width() + + Return the width of the widget in pixels. + When a window is first created its width is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqwidth`. + + .. method:: winfo_x() + + Return the *x* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: winfo_y() + + Return the *y* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: info_patchlevel() + + Return the Tcl/Tk patch level as a named tuple with the same five fields + as :data:`sys.version_info`: *major*, *minor*, *micro*, *releaselevel* + and *serial*. + *releaselevel* is ``'alpha'``, ``'beta'`` or ``'final'``. + Converting it to a string gives the version in the usual Tcl/Tk notation, + for example ``'9.0.3'`` for a final release or ``'9.1b2'`` for a + pre-release. + + .. versionadded:: 3.11 + + + +.. class:: Wm() + + The :class:`!Wm` mixin provides access to the window manager, allowing an + application to control such things as the title, geometry and icon of a + top-level window, the way it is resized, and how it responds to window + manager protocols. + It is mixed into :class:`Tk` and :class:`Toplevel`, so its methods are + available on every top-level window. + Each method has two equivalent spellings: a short name and a + ``wm_``-prefixed name (for example, :meth:`title` and :meth:`wm_title`). + + .. method:: wm_aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + :no-typesetting: + + .. method:: aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + + Constrain the aspect ratio (the ratio of width to height) of the window. + If all four arguments are given, the window manager keeps the ratio + between ``minNumer/minDenom`` and ``maxNumer/maxDenom``; passing empty + strings removes any existing restriction. + With no arguments, return a tuple of the four current values, or an empty + string if no aspect restriction is in effect. + :meth:`wm_aspect` is an alias of :meth:`!aspect`. + + .. method:: wm_attributes(*args, return_python_dict=False, **kwargs) + :no-typesetting: + + .. method:: attributes(*args, return_python_dict=False, **kwargs) + + Query or set platform-specific attributes of the window. + With no arguments, return the platform-specific flags and their values; + pass *return_python_dict* as true to get them as a dictionary. + A single option name such as ``'alpha'`` returns the value of that + option, and options are set using keyword arguments (``alpha=0.5``). + + The available attributes differ by platform. + All platforms support: + + *alpha* + The window's opacity, from ``0.0`` (fully transparent) to ``1.0`` + (opaque). + Where transparency is unsupported the value stays at ``1.0``. + + *appearance* + Whether the window is rendered in dark mode on Windows and macOS: + ``'auto'``, ``'light'`` or ``'dark'`` (this has no effect on X11). + + *fullscreen* + Whether the window takes up the entire screen and has no borders. + + *topmost* + Whether the window is displayed above all other windows. + + Windows additionally supports: + + *disabled* + Whether the window is in a disabled state. + + *toolwindow* + Whether the window uses the tool window style. + + *transparentcolor* + The color that is made fully transparent, or an empty string for none. + + macOS additionally supports: + + *class* + Whether the underlying Aqua window is an ``nswindow`` or an + ``nspanel``; this can only be set before the window is created. + + *modified* + The modification state shown by the window's close button and proxy + icon. + + *notify* + Whether the application's dock icon bounces to request attention. + + *stylemask* + The style mask of the underlying Aqua window, given as a list of bit + names such as ``titled`` or ``resizable``. + + *tabbingid* + The identifier of the tab group that the window belongs to. + + *tabbingmode* + Whether the window may be opened as a tab: ``'auto'``, ``'preferred'`` + or ``'disallowed'``. + + *titlepath* + The path of the file represented by the window's proxy icon. + + *transparent* + Whether the content area is transparent and the window shadow is + turned off. + + X11 additionally supports: + + *type* + The window type, or a list of types in order of preference, that the + window manager should use to interpret the window, such as + ``'dialog'`` or ``'splash'``. + + *zoomed* + Whether the window is maximized. + + .. note:: + + Tk 8.6 added the *type* attribute, and Tk 9.0 added the *appearance*, + *class*, *stylemask*, *tabbingid* and *tabbingmode* attributes. + + On X11 changes are applied asynchronously, so a queried value may not yet + reflect the most recent request. + :meth:`wm_attributes` is an alias of :meth:`!attributes`. + + .. versionchanged:: 3.13 + A single attribute may now be queried by name without the leading + ``-``, and attributes may be set using keyword arguments. + The *return_python_dict* parameter was added. + + .. deprecated:: 3.13 + Setting an attribute by passing the option name (with a leading + ``-``) and its value as two positional arguments, as in + ``w.attributes('-alpha', 0.5)``, is deprecated; use keyword arguments + instead. + + + .. method:: wm_client(name=None) + :no-typesetting: + + .. method:: client(name=None) + + Store *name*, which should be the name of the host on which the + application is running, in the window's ``WM_CLIENT_MACHINE`` property + for use by the window or session manager. + An empty string deletes the property. + With no argument, return the last name set, or an empty string. + :meth:`wm_client` is an alias of :meth:`!client`. + + .. method:: wm_colormapwindows(*wlist) + :no-typesetting: + + .. method:: colormapwindows(*wlist) + + Manipulate the ``WM_COLORMAP_WINDOWS`` property, which tells the window + manager about windows that have private colormaps. + If *wlist* is given, overwrite the property with those windows (their + order is a priority order for installing colormaps). + With no arguments, return the list of windows currently named in the + property. + :meth:`wm_colormapwindows` is an alias of :meth:`!colormapwindows`. + + .. method:: wm_command(value=None) + :no-typesetting: + + .. method:: command(value=None) + + Store *value* in the window's ``WM_COMMAND`` property for use by the + window or session manager; it should be a list giving the words of the + command used to invoke the application. + An empty string deletes the property. + With no argument, return the last value set, or an empty string. + :meth:`wm_command` is an alias of :meth:`!command`. + + .. method:: wm_deiconify() + :no-typesetting: + + .. method:: deiconify() + + Display the window in normal (non-iconified) form by mapping it. + If the window has never been mapped, this ensures it appears de-iconified + when it is first mapped. + On Windows the window is also raised and given the focus. + :meth:`wm_deiconify` is an alias of :meth:`!deiconify`. + + .. method:: wm_focusmodel(model=None) + :no-typesetting: + + .. method:: focusmodel(model=None) + + Set or query the focus model for the window. + *model* is either ``'active'`` (the window claims the input focus for + itself or its descendants, even when the focus is in another application) + or ``'passive'`` (the window relies on the window manager to give it the + focus). + With no argument, return the current model. + The default is ``'passive'``, which is what the :meth:`!focus` command + assumes. + :meth:`wm_focusmodel` is an alias of :meth:`!focusmodel`. + + .. method:: wm_forget(window) + :no-typesetting: + + .. method:: forget(window) + + Unmap *window* from the screen so that it is no longer managed by the + window manager. + A :class:`Toplevel` is then treated like a :class:`Frame`, although its + ``-menu`` configuration is remembered and the menu reappears if the + widget is managed again. + :meth:`wm_forget` is an alias of :meth:`!forget`. + + Not to be confused with :meth:`Pack.forget`. + + .. versionadded:: 3.3 + + .. method:: wm_frame() + :no-typesetting: + + .. method:: frame() + + Return the platform-specific window identifier for the outermost + decorative frame containing the window, if the window manager has + reparented it into such a frame; otherwise return the identifier of the + window itself. + :meth:`wm_frame` is an alias of :meth:`!frame`. + + .. method:: wm_geometry(newGeometry=None) + :no-typesetting: + + .. method:: geometry(newGeometry=None) + + Set or query the geometry of the window. + *newGeometry* has the form ``=widthxheight+x+y``, where any of ``=``, + ``widthxheight`` and the ``+x+y`` position may be omitted. + *width* and *height* are in pixels (or grid units for a gridded window); + a position preceded by ``+`` is measured from the left or top edge of the + screen and one preceded by ``-`` from the right or bottom edge. + An empty string cancels any user-specified geometry, letting the window + revert to its natural size. + With no argument, return the current geometry as a string of the form + ``'200x200+10+10'``. + :meth:`wm_geometry` is an alias of :meth:`!geometry`. + + .. method:: wm_grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + :no-typesetting: + + .. method:: grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + + Manage the window as a gridded window and define the relationship between + grid units and pixels. + *baseWidth* and *baseHeight* are the numbers of grid units for the + window's internally requested size, and *widthInc* and *heightInc* are + the pixel sizes of a horizontal and vertical grid unit. + Empty strings turn off gridded management. + With no arguments, return a tuple of the four current values, or an empty + string if the window is not gridded. + :meth:`wm_grid` is an alias of :meth:`!grid`. + + Not to be confused with the grid geometry manager :meth:`Grid.grid`. + + .. method:: wm_group(pathName=None) + :no-typesetting: + + .. method:: group(pathName=None) + + Set or query the leader of a group of related windows. + *pathName* gives the path name of the group leader; the window manager + may, for example, unmap all windows in the group when the leader is + iconified. + An empty string removes the window from any group. + With no argument, return the path name of the current group leader, or an + empty string. + :meth:`wm_group` is an alias of :meth:`!group`. + + .. method:: wm_iconbadge(badge) + :no-typesetting: + + .. method:: iconbadge(badge) + + Set a badge for the window's icon, intended for display in the Dock + (macOS), taskbar (Windows) or app panel (X11). + *badge* may be a positive integer (for example a count of unread + messages) or an exclamation point to denote that attention is needed; + an empty string removes the badge. + On X11 the variable ``::tk::icons::base_icon(window)`` must be set to the + window's icon image for the badge to appear. + :meth:`wm_iconbadge` is an alias of :meth:`!iconbadge`. + + .. versionadded:: next + + Requires Tk 9.0 or newer. + + .. method:: wm_iconbitmap(bitmap=None, default=None) + :no-typesetting: + + .. method:: iconbitmap(bitmap=None, default=None) + + Set or query the bitmap used by the window manager for the window's icon. + *bitmap* names a bitmap in one of the standard forms accepted by Tk; an + empty string cancels the current icon bitmap. + With no argument, return the name of the current icon bitmap, or an empty + string. + On Windows the *default* argument names an icon (for example an ``.ico`` + file) applied to all top-level windows that have no icon of their own. + :meth:`wm_iconbitmap` is an alias of :meth:`!iconbitmap`. + + .. method:: wm_iconify() + :no-typesetting: + + .. method:: iconify() + + Iconify the window. + If the window has not yet been mapped for the first time, arrange for it + to appear in the iconified state when it is eventually mapped. + :meth:`wm_iconify` is an alias of :meth:`!iconify`. + + .. method:: wm_iconmask(bitmap=None) + :no-typesetting: + + .. method:: iconmask(bitmap=None) + + Set or query the bitmap used as a mask for the icon (see + :meth:`iconbitmap`). + Where the mask is zero no icon is displayed; where it is one, the + corresponding bits of the icon bitmap are shown. + An empty string cancels the current mask. + With no argument, return the name of the current icon mask, or an empty + string. + :meth:`wm_iconmask` is an alias of :meth:`!iconmask`. + + .. method:: wm_iconname(newName=None) + :no-typesetting: + + .. method:: iconname(newName=None) + + Set or query the name displayed by the window manager inside the window's + icon. + With no argument, return the current icon name, or an empty string if + none has been set (in which case the window manager normally displays the + window's title). + :meth:`wm_iconname` is an alias of :meth:`!iconname`. + + .. method:: wm_iconphoto(default=False, *images) + :no-typesetting: + + .. method:: iconphoto(default=False, *images) + + Set the titlebar icon for the window from one or more :class:`PhotoImage` + objects given in *images*. + Several images of different sizes (for example 16x16 and 32x32) may be + supplied so that the window manager can choose an appropriate one. + The image data is taken as a snapshot at the time of the call; later + changes to the images are not reflected. + If *default* is true, the icon is also applied to all top-level windows + created in the future. + On macOS only the first image is used. + :meth:`wm_iconphoto` is an alias of :meth:`!iconphoto`. + + .. versionadded:: 3.3 + + .. method:: wm_iconposition(x=None, y=None) + :no-typesetting: + + .. method:: iconposition(x=None, y=None) + + Set or query a hint to the window manager about where the window's icon + should be positioned. + Empty strings cancel an existing hint. + With no arguments, return a tuple of the two current values, or an empty + string if no hint is in effect. + :meth:`wm_iconposition` is an alias of :meth:`!iconposition`. + + .. method:: wm_iconwindow(pathName=None) + :no-typesetting: + + .. method:: iconwindow(pathName=None) + + Set or query the window used as the icon for the window. + When the window is iconified, *pathName* is mapped to serve as its icon + and unmapped again when it is de-iconified. + An empty string cancels the association. + With no argument, return the path name of the current icon window, or an + empty string. + Not all window managers support icon windows, and the concept is + meaningless on non-X11 platforms. + :meth:`wm_iconwindow` is an alias of :meth:`!iconwindow`. + + .. method:: wm_manage(widget) + :no-typesetting: + + .. method:: manage(widget) + + Make *widget* a stand-alone top-level window, decorated by the window + manager with a title bar and so on. + Only :class:`Frame`, :class:`LabelFrame` and :class:`Toplevel` widgets + may be used; passing any other widget type raises an error. + :meth:`wm_manage` is an alias of :meth:`!manage`. + + .. versionadded:: 3.3 + + .. method:: wm_maxsize(width=None, height=None) + :no-typesetting: + + .. method:: maxsize(width=None, height=None) + + Set or query the maximum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no larger than *width* and + *height*. + With no arguments, return a tuple of the current maximum width and + height. + The maximum size defaults to the size of the screen. + :meth:`wm_maxsize` is an alias of :meth:`!maxsize`. + + .. method:: wm_minsize(width=None, height=None) + :no-typesetting: + + .. method:: minsize(width=None, height=None) + + Set or query the minimum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no smaller than *width* and + *height*. + With no arguments, return a tuple of the current minimum width and + height. + The minimum size defaults to one pixel in each dimension. + :meth:`wm_minsize` is an alias of :meth:`!minsize`. + + .. method:: wm_overrideredirect(boolean=None) + :no-typesetting: + + .. method:: overrideredirect(boolean=None) + + Set or query the override-redirect flag for the window. + When this flag is set, the window is ignored by the window manager: it is + not reparented into a decorative frame and the user cannot manipulate it + through the usual window manager controls. + With no argument, return a boolean indicating whether the flag is set. + The flag is reliably honored only when the window is first mapped or + remapped from the withdrawn state. + :meth:`wm_overrideredirect` is an alias of :meth:`!overrideredirect`. + + .. method:: wm_positionfrom(who=None) + :no-typesetting: + + .. method:: positionfrom(who=None) + + Set or query the source of the window's current position. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + position was requested by the program or by the user; an empty string + cancels the current source. + With no argument, return the current source, or an empty string if none + has been set. + Tk automatically sets the source to ``'user'`` when :meth:`geometry` is + called, unless it has been set explicitly to ``'program'``. + :meth:`wm_positionfrom` is an alias of :meth:`!positionfrom`. + + .. method:: wm_protocol(name=None, func=None) + :no-typesetting: + + .. method:: protocol(name=None, func=None) + + Register *func* as the handler for the window manager protocol *name*, an + atom such as ``'WM_DELETE_WINDOW'``, ``'WM_SAVE_YOURSELF'`` or + ``'WM_TAKE_FOCUS'``; *func* is then called whenever the window manager + sends a message of that protocol. + Tk installs a default ``WM_DELETE_WINDOW`` handler that destroys the + window, which this method can replace. + If *func* is an empty string, the handler is removed. + With only *name*, return the name of its registered handler command, or + an empty string if none is set (the default ``WM_DELETE_WINDOW`` handler + is not reported); with no arguments, return a tuple of the protocols that + currently have handlers. + :meth:`wm_protocol` is an alias of :meth:`!protocol`. + + .. method:: wm_resizable(width=None, height=None) + :no-typesetting: + + .. method:: resizable(width=None, height=None) + + Control whether the user may interactively resize the window. + *width* and *height* are boolean values that determine whether the + window's width and height may be changed. + With no arguments, return a tuple of two ``0``/``1`` values indicating + whether each dimension is currently resizable. + By default a window is resizable in both dimensions. + :meth:`wm_resizable` is an alias of :meth:`!resizable`. + + .. method:: wm_sizefrom(who=None) + :no-typesetting: + + .. method:: sizefrom(who=None) + + Set or query the source of the window's current size. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + size was requested by the program or by the user; an empty string cancels + the current source. + With no argument, return the current source, or an empty string if none + has been set. + :meth:`wm_sizefrom` is an alias of :meth:`!sizefrom`. + + .. method:: wm_stackorder(relation=None, window=None) + :no-typesetting: + + .. method:: stackorder(relation=None, window=None) + + Query the stacking order of top-level windows. + With no arguments, return a list of the mapped top-level widgets in + stacking order, from lowest to highest, recursively including this + window's top-level children. + If *relation* is ``'isabove'`` or ``'isbelow'`` and *window* is another + top-level, return ``True`` if this window is respectively above or below + *window* in the stacking order, and ``False`` otherwise. + :meth:`wm_stackorder` is an alias of :meth:`!stackorder`. + + .. versionadded:: next + + .. method:: wm_state(newstate=None) + :no-typesetting: + + .. method:: state(newstate=None) + + Set or query the state of the window. + With no argument, return the current state: one of ``'normal'``, + ``'iconic'``, ``'withdrawn'``, ``'icon'`` or, on Windows and macOS only, + ``'zoomed'``. + ``'iconic'`` refers to a window that has been iconified, while ``'icon'`` + refers to a window serving as the icon for another window (see + :meth:`iconwindow`); the ``'icon'`` state cannot be set. + :meth:`wm_state` is an alias of :meth:`!state`. + + Not to be confused with :meth:`ttk.Widget.state + `. + + .. method:: wm_title(string=None) + :no-typesetting: + + .. method:: title(string=None) + + Set or query the title for the window, which the window manager should + display in the window's title bar. + With no argument, return the current title. + The title defaults to the window's name. + :meth:`wm_title` is an alias of :meth:`!title`. + + .. method:: wm_transient(master=None) + :no-typesetting: + + .. method:: transient(master=None) + + Mark the window as a transient window (such as a pull-down menu or + dialog) working on behalf of *master*, the path name of another top-level + window. + An empty string clears the transient status. + With no argument, return the path name of the current master, or an empty + string. + A transient window mirrors state changes in its master and may be + decorated differently by the window manager; it is an error to make a + window a transient of itself. + :meth:`wm_transient` is an alias of :meth:`!transient`. + + .. method:: wm_withdraw() + :no-typesetting: + + .. method:: withdraw() + + Withdraw the window from the screen, unmapping it and causing the window + manager to forget about it. + If the window has never been mapped, it is instead mapped in the + withdrawn state. + It is sometimes necessary to withdraw a window and then re-map it (for + example with :meth:`deiconify`) to make some window managers notice + changes to window attributes. + :meth:`wm_withdraw` is an alias of :meth:`!withdraw`. + + +.. class:: Pack() + + Geometry manager that arranges widgets by packing them against the sides of + their container. + The :class:`!Pack` mix-in is inherited by all widgets (through + :class:`Widget`) and provides the methods for managing a widget with the + *pack* geometry manager. + See also :ref:`pack-the-packer`. + + .. note:: + + :class:`Pack`, :class:`Place` and :class:`Grid` all define the short + method names :meth:`!forget`, :meth:`!info`, :meth:`!slaves`, + :meth:`!content` and :meth:`!propagate`. + On a widget the bare names resolve to the *pack* manager's versions, + since :class:`Pack` and :class:`Misc` precede :class:`Place` and + :class:`Grid` in the method resolution order, + whatever manager actually manages the widget; + and :meth:`!configure`/:meth:`!config` configure the widget's options, + not its geometry. + Use the explicit ``pack_*``, ``grid_*`` and ``place_*`` methods + (and ``pack``, ``grid``, ``place`` for geometry configuration) + to act on a specific geometry manager. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: pack_configure(cnf={}, **kw) + pack(cnf={}, **kw) + + Pack the widget inside its container, positioning it relative to the + siblings already packed there. + The supported options are: + + *side* + Which side of the container to pack the widget against: ``'top'`` (the + default), ``'bottom'``, ``'left'`` or ``'right'``. + + *fill* + Whether to stretch the widget to fill its parcel: ``'none'`` (the + default), ``'x'``, ``'y'`` or ``'both'``. + + *expand* + Whether the widget should expand to consume any extra space in its + container (a boolean, default false). + + *anchor* + Where to position the widget in its parcel when the parcel is larger + than the widget: an anchor such as ``'n'`` or ``'sw'`` (default + ``'center'``). + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *after* + Pack the widget after the given widget in the packing order, using the + same container. + + *before* + Pack the widget before the given widget in the packing order, using + the same container. + + *in_* + The container in which to pack the widget; it defaults to the parent + widget. + + :meth:`pack`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!pack_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: pack_forget() + + Unmap the widget and remove it from the packing order, forgetting its + packing options. + It can be packed again later with :meth:`pack_configure`. + :meth:`forget` is an alias of :meth:`!pack_forget`, + except on :class:`PanedWindow`, + :class:`ttk.Notebook ` and + :class:`ttk.PanedWindow `, + which provide their own :meth:`!forget` method. + + Not to be confused with :meth:`Wm.forget`. + + .. method:: info() + :no-typesetting: + + .. method:: pack_info() + + Return a dictionary of the widget's current packing options. + :meth:`info` is an alias of :meth:`!pack_info`. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Same as :meth:`Misc.pack_propagate`, treating this widget as a container: + enable or disable geometry propagation. + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Same as :meth:`Misc.pack_slaves`: return the list of widgets packed in + this widget. + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: pack_content() + + Same as :meth:`Misc.pack_content`. + :meth:`content` is an alias of :meth:`!pack_content`. + + .. versionadded:: 3.15 + + +.. class:: Place() + + Geometry manager that places widgets at explicit positions and sizes within + their container. + The :class:`!Place` mix-in is inherited by all widgets (through + :class:`Widget`). + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: place_configure(cnf={}, **kw) + place(cnf={}, **kw) + + Place the widget inside its container at an absolute or relative + position. + The supported options are: + + *x*, *y* + The absolute horizontal and vertical position of the widget's anchor + point, as a screen distance (default ``0``). + + *relx*, *rely* + The horizontal and vertical position of the widget's anchor point as a + fraction of the container's width and height, where ``0.0`` is the + left or top edge and ``1.0`` is the right or bottom edge. + If both the absolute and the relative option are given, their values + are summed. + + *anchor* + Which point of the widget is placed at the given position: an anchor + such as ``'n'`` or ``'se'`` (default ``'nw'``). + + *width*, *height* + The absolute width and height of the widget, as a screen distance. + By default the widget's requested size is used. + + *relwidth*, *relheight* + The width and height of the widget as a fraction of the container's + width and height. + If both the absolute and the relative option are given, their values + are summed. + + *bordermode* + How the container's border affects placement: ``'inside'`` (the + default) measures the area inside the border, ``'outside'`` measures + the area including the border, and ``'ignore'`` uses the official X + area. + + *in_* + The container relative to which the widget is placed; it must be the + widget's parent or a descendant of the parent, and defaults to the + parent. + + :meth:`place`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!place_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: place_forget() + + Unmap the widget and remove it from the placement, forgetting its place + options. + + .. method:: info() + :no-typesetting: + + .. method:: place_info() + + Return a dictionary of the widget's current place options. + + .. method:: slaves() + :no-typesetting: + + .. method:: place_slaves() + + Same as :meth:`Misc.place_slaves`: return the list of widgets placed in + this widget. + + .. method:: content() + :no-typesetting: + + .. method:: place_content() + + Same as :meth:`Misc.place_content`. + + .. versionadded:: 3.15 + + +.. class:: Grid() + + Geometry manager that arranges widgets in a two-dimensional grid of rows and + columns within their container. + The :class:`!Grid` mix-in is inherited by all widgets (through + :class:`Widget`). + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: grid_configure(cnf={}, **kw) + grid(cnf={}, **kw) + + Position the widget in a cell of its container's grid. + + Not to be confused with :meth:`Wm.grid`. + + The supported options are: + + *row*, *column* + The row and column of the cell to place the widget in, counting from + ``0``. + *column* defaults to the column after the previous widget placed in + the same :meth:`!grid_configure` call (or ``0``), and *row* defaults + to the next empty row. + + *rowspan*, *columnspan* + The number of rows and columns the widget should span (default ``1``). + + *sticky* + How to position or stretch the widget when its cell is larger than the + widget: a string containing zero or more of the characters ``'n'``, + ``'s'``, ``'e'`` and ``'w'``, naming the cell sides the widget sticks + to. + Specifying both ``'n'`` and ``'s'`` (or ``'e'`` and ``'w'``) stretches + the widget to fill the height (or width) of the cell. + The default is ``''``, which centers the widget at its requested size. + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *in_* + The container in whose grid to place the widget; it defaults to the + parent widget. + + :meth:`grid`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!grid_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: grid_forget() + + Unmap the widget and remove it from the grid, forgetting its grid + options. + + .. method:: grid_remove() + + Unmap the widget and remove it from the grid, but remember its grid + options so that it is restored to the same cell if it is gridded again. + + .. method:: info() + :no-typesetting: + + .. method:: grid_info() + + Return a dictionary of the widget's current grid options. + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Same as :meth:`Misc.grid_bbox`. + :meth:`bbox` is an alias of :meth:`!grid_bbox`, + except on :class:`Canvas`, :class:`Listbox`, :class:`Spinbox`, + :class:`Text`, :class:`ttk.Entry ` and + :class:`ttk.Treeview `, + which provide their own :meth:`!bbox` method. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_columnconfigure`: query or set the options (such + as *weight*, *minsize*, *pad* and *uniform*) of a grid column. + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_rowconfigure`: query or set the options of a + grid row. + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: location(x, y) + :no-typesetting: + + .. method:: grid_location(x, y) + + Same as :meth:`Misc.grid_location`: return the ``(column, row)`` of the + cell that covers the pixel at *x*, *y*. + :meth:`location` is an alias of :meth:`!grid_location`. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Same as :meth:`Misc.grid_size`: return a ``(columns, rows)`` tuple giving + the size of the grid. + :meth:`size` is an alias of :meth:`!grid_size`, + except on :class:`Listbox` and + :class:`ttk.Treeview `, + which provide their own :meth:`!size` method. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: grid_propagate() + grid_propagate(flag) + + Same as :meth:`Misc.grid_propagate`. + + .. method:: slaves(row=None, column=None) + :no-typesetting: + + .. method:: grid_slaves(row=None, column=None) + + Same as :meth:`Misc.grid_slaves`: return the widgets managed in the grid, + optionally restricted to a *row* and/or *column*. + + .. method:: content(row=None, column=None) + :no-typesetting: + + .. method:: grid_content(row=None, column=None) + + Same as :meth:`Misc.grid_content`. + + .. versionadded:: 3.15 + + +.. class:: XView() + + Mix-in providing the horizontal-scrolling interface shared by widgets such + as :class:`Entry`, :class:`Canvas`, :class:`Listbox`, :class:`Text` and + :class:`Spinbox`. + A widget's :meth:`xview` method is registered as the *command* of a + horizontal :class:`Scrollbar`. + + .. method:: xview(*args) + + Query or change the horizontal position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``xview`` widget command and + are usually generated by a scrollbar; :meth:`xview_moveto` and + :meth:`xview_scroll` provide a more convenient interface. + + .. method:: xview_moveto(fraction) + + Adjust the view so that *fraction* of the total width of the document is + off-screen to the left. + *fraction* is a number between 0 and 1. + + .. method:: xview_scroll(number, what) + + Shift the view left or right by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + left and a positive one scrolls right. + + +.. class:: YView() + + Mix-in providing the vertical-scrolling interface shared by widgets such as + :class:`Canvas`, :class:`Listbox` and :class:`Text`. + A widget's :meth:`yview` method is registered as the *command* of a vertical + :class:`Scrollbar`. + + .. method:: yview(*args) + + Query or change the vertical position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``yview`` widget command, + usually generated by a scrollbar; :meth:`yview_moveto` and + :meth:`yview_scroll` provide a more convenient interface. + + .. method:: yview_moveto(fraction) + + Adjust the view so that *fraction* of the total height of the document is + off-screen above the top. + *fraction* is a number between 0 and 1. + + .. method:: yview_scroll(number, what) + + Shift the view up or down by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + up and a positive one scrolls down. + + +.. class:: BaseWidget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for all widgets. + It inherits from :class:`Misc` and adds the machinery that creates the + underlying Tk widget; application code normally uses :class:`Widget` or a + concrete widget class rather than instantiating :class:`!BaseWidget` + directly. + + .. method:: destroy() + + Destroy this widget and all of its children, removing the corresponding + Tk widgets and deleting the associated Tcl commands. + + +.. class:: Widget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for the standard widgets. + It combines :class:`BaseWidget` with the geometry-manager mix-ins + :class:`Pack`, :class:`Place` and :class:`Grid`, so that every widget can be + managed by any of the three geometry managers. + The concrete widget classes (:class:`Button`, :class:`Label`, and so on) + derive from :class:`!Widget`. + + +Toplevel widgets +^^^^^^^^^^^^^^^^ + +.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) + + Construct a toplevel Tk widget, which is usually the main window of an + application, and initialize a Tcl interpreter for this widget. Each + instance has its own associated Tcl interpreter. + Inherits from :class:`Misc` and :class:`Wm`. + + To create a Tcl interpreter without initializing the Tk subsystem, use the + :func:`Tcl` factory function instead. + + The :class:`Tk` class is typically instantiated using all default values. + However, the following keyword arguments are currently recognized: + + *screenName* + When given (as a string), sets the :envvar:`DISPLAY` environment + variable. (X11 only) + *baseName* + Name of the profile file. By default, *baseName* is derived from the + program name (``sys.argv[0]``). + *className* + Name of the widget class. Used as a profile file and also as the name + with which Tcl is invoked (*argv0* in *interp*). + *useTk* + If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` + function sets this to ``False``. + *sync* + If ``True``, execute all X server commands synchronously, so that errors + are reported immediately. Can be used for debugging. (X11 only) + *use* + Specifies the *id* of the window in which to embed the application, + instead of it being created as an independent toplevel window. *id* must + be specified in the same way as the value for the -use option for + toplevel widgets (that is, it has a form like that returned by + :meth:`~Misc.winfo_id`). + + Note that on some platforms this will only work correctly if *id* refers + to a Tk frame or toplevel that has its -container option enabled. + + :class:`Tk` reads and interprets profile files, named + :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl + interpreter and calls :func:`exec` on the contents of + :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the + profile files is the :envvar:`HOME` environment variable or, if that + isn't defined, then :data:`os.curdir`. + + .. attribute:: tk + + The Tk application object created by instantiating :class:`Tk`. This + provides access to the Tcl interpreter. Each widget that is attached + the same instance of :class:`Tk` has the same value for its :attr:`tk` + attribute. + + .. attribute:: master + + The widget object that contains this widget. + For :class:`Tk`, the :attr:`!master` is :const:`None` because it is the + main window. + The terms *master* and *parent* are similar and sometimes used + interchangeably as argument names; however, calling + :meth:`~Misc.winfo_parent` returns a string of the widget name whereas + :attr:`!master` returns the object. + *parent*/*child* reflects the tree-like relationship while *master* (or + *container*)/*content* reflects the container structure. + + .. attribute:: children + + The immediate descendants of this widget as a :class:`dict` with the + child widget names as the keys and the child instance objects as the + values. + + .. method:: destroy() + + Destroy this and all descendant widgets and, for the main window, end the + connection to the underlying Tcl interpreter. + + .. method:: loadtk() + + Finish loading and initializing the Tk subsystem. + This is needed only when the interpreter was created without Tk (for + example through :func:`Tcl`); it is called automatically when *useTk* is + true. + + .. method:: readprofile(baseName, className) + + Read and source the user's profile files :file:`.{className}.tcl` and + :file:`.{baseName}.tcl` into the Tcl interpreter, and execute the + corresponding :file:`.{className}.py` and :file:`.{baseName}.py` files. + This is called during initialization; see the description of the + constructor above. + + .. method:: report_callback_exception(exc, val, tb) + + Report a callback exception. + This is called when an exception propagates out of a Tkinter callback; + *exc*, *val* and *tb* are the exception type, value and traceback as + returned by :func:`sys.exc_info`. + The default implementation prints a traceback to :data:`sys.stderr`. + It can be overridden to customize error handling, for example to display + the traceback in a dialog. + + +.. class:: Toplevel(master=None, cnf={}, **kw) + + A :class:`!Toplevel` widget is a top-level window, similar to a + :class:`Frame` except that its X parent is the root window of a screen + rather than its logical parent. + Its primary purpose is to serve as a container for dialog boxes and other + collections of widgets; its only visible features are its background and an + optional 3-D border. + Notable options include *menu*, which installs a :class:`Menu` as the + window's menubar. + Inherits from :class:`BaseWidget` and :class:`Wm`, so a toplevel is managed + by the window manager. + Refer to the Tk ``toplevel`` manual page for the full list of options. + + +Widget classes +^^^^^^^^^^^^^^ + +.. class:: Button(master=None, cnf={}, **kw) + + A :class:`!Button` widget displays a textual string, bitmap or image and + invokes a command when the user presses it (by clicking mouse button 1 over + the button or, when the button has focus, by pressing the space key). + Inherits from :class:`Widget`. + In addition to the standard widget options, a button accepts the options + documented in the Tk ``button`` manual page, such as *command* (the callback + invoked when the button is pressed), *textvariable*, *state* and *default*. + + .. method:: invoke() + + Invoke the command associated with the button, if there is one, and + return its result, or an empty string if no command is associated with + the button. + This is ignored if the button's state is ``disabled``. + + .. method:: flash() + + Flash the button by redisplaying it several times, alternating between + the active and normal colors. + At the end of the flash the button is left in the same normal or active + state as when the method was called. + This is ignored if the button's state is ``disabled``. + + +.. class:: Canvas(master=None, cnf={}, **kw) + + A :class:`!Canvas` widget implements structured graphics. + It displays any number of *items*, such as arcs, lines, ovals, polygons, + rectangles, text, bitmaps, images and embedded windows, which may be drawn, + moved, re-colored and bound to events. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``canvas`` manual page for the full list of widget and item + options. + + Each item has a unique integer *id*, assigned when it is created, and zero + or more string *tags*. + A tag is an arbitrary string that does not have the form of an integer; the + same tag may be shared by many items, which makes tags convenient for + grouping items. + The special tag ``'all'`` matches every item in the canvas, and + ``'current'`` matches the topmost item under the mouse pointer. + Most methods take a *tagOrId* argument that may be an integer id naming a + single item, or a tag naming zero or more items; as described in the Tk + ``canvas`` manual page, a tag may also be a logical expression of tags + combined with the operators ``&&``, ``||``, ``^``, ``!`` and parentheses. + When a method that operates on a single item is given a *tagOrId* matching + several items, it normally uses the lowest matching item in the display + list. + + The items are kept in a *display list* that determines drawing order: items + later in the list are drawn on top of earlier ones. + A newly created item is placed at the top of the list; the order can be + changed with :meth:`tag_raise` and :meth:`tag_lower`. + + .. method:: create_arc(*args, **kw) + create_bitmap(*args, **kw) + create_image(*args, **kw) + create_line(*args, **kw) + create_oval(*args, **kw) + create_polygon(*args, **kw) + create_rectangle(*args, **kw) + create_text(*args, **kw) + create_window(*args, **kw) + + Create a new item of the corresponding type and return its integer id. + Each method is called as ``create_TYPE(coord..., **options)``: the + leading positional arguments give the coordinates that define the item + (as separate numbers, as a single sequence of numbers, or as coordinate + pairs), and the keyword arguments set item-specific options. + Coordinates and screen distances may be given as numbers (interpreted as + pixels) or as strings with a unit suffix (``'m'``, ``'c'``, ``'i'`` or + ``'p'`` for millimetres, centimetres, inches or printer's points), but + are always stored and returned in pixels. + + The item types are: ``arc`` (an arc-shaped region that is a section of an + oval, defined by two diagonally opposite corners ``x1, y1, x2, y2`` of + the enclosing rectangle); ``bitmap`` (a two-color bitmap positioned at a + point ``x, y``); ``image`` (a Tk image positioned at a point ``x, y``); + ``line`` (a line or curve through the points ``x1, y1, ..., xn, yn``); + ``oval`` (a circle or ellipse inscribed in the rectangle + ``x1, y1, x2, y2``); ``polygon`` (a closed polygon through the points + ``x1, y1, ..., xn, yn``); ``rectangle`` (a rectangle with corners + ``x1, y1, x2, y2``); ``text`` (a string of text positioned at a point + ``x, y``); and ``window`` (a child widget embedded in the canvas at a + point ``x, y``, specified with the *window* option). + + Most item types accept a common set of *standard item options*, plus a + few options specific to each type. + Option names are passed as keyword arguments, without the leading + hyphen. + + The standard item options are: + + *fill* + The color used to fill the item's interior, or to draw a *line* + item or the characters of a *text* item. + An empty string (the default for all types except *line* and *text*) + leaves the item unfilled. + + *outline* + The color used to draw the item's outline. + An empty string draws no outline. + + *width* + The width of the outline, defaulting to ``1.0``. + Has no effect if *outline* is empty. + + *dash* + A dash pattern for the outline, given either as a sequence of + segment lengths in pixels or as a string of the characters + ``'.'``, ``','``, ``'-'``, ``'_'`` and space. + An empty pattern (the default) draws a solid outline. + + *dashoffset* + The starting offset in pixels into the *dash* pattern. + Ignored if there is no *dash* pattern. + + *stipple* + A bitmap used as a stipple pattern when filling the item. + Only well supported on X11. + + *outlinestipple* + A bitmap used as a stipple pattern when drawing the outline. + Has no effect if *outline* is empty. + + *offset*, *outlineoffset* + The offset of the fill and outline stipple patterns, given as + ``'x,y'`` or as a side such as ``'n'``, ``'se'`` or ``'center'``. + Stipple offsets are only supported on X11. + + *state* + Overrides the canvas state for this item; one of ``'normal'``, + ``'disabled'`` or ``'hidden'``. + + *tags* + A single tag or a sequence of tags to associate with the item, + replacing any existing tags. + + Many of these options have *active...* and *disabled...* variants + (such as *activefill*, *disabledfill*, *activewidth*, *disableddash*, + *activeoutline*, *disabledstipple*) that override the base option when + the item is the active item (under the mouse pointer) or is in the + disabled state. + + The following item types support additional options. + + For ``arc`` items: + + *start* + The start of the arc's angular range, in degrees measured + counter-clockwise from the 3-o'clock position. + + *extent* + The size of the angular range, in degrees counter-clockwise from + *start*. + + *style* + How the arc is drawn: ``'pieslice'`` (the default), ``'chord'`` or + ``'arc'``. + + For ``line`` items: + + *arrow* + Where to draw arrowheads: ``'none'`` (the default), ``'first'``, + ``'last'`` or ``'both'``. + + *arrowshape* + A sequence of three distances describing the shape of the + arrowheads. + + *capstyle* + How line ends are drawn: ``'butt'`` (the default), ``'projecting'`` + or ``'round'``. + + *joinstyle* + How line vertices are drawn: ``'round'`` (the default), ``'bevel'`` + or ``'miter'``. + + *smooth* + The smoothing method: a false value (the default) for no smoothing, + or ``'true'``/``'bezier'`` or ``'raw'`` to draw the line as a curve. + + *splinesteps* + The number of line segments approximating each spline when + *smooth* is enabled. + + For ``polygon`` items: + + *joinstyle*, *smooth*, *splinesteps* + As for ``line`` items, applied to the polygon's outline. + + For ``text`` items: + + *text* + The string to display; newline characters start new lines. + + *font* + The font used for the text. + + *justify* + How lines are justified: ``'left'`` (the default), ``'right'`` or + ``'center'``. + + *anchor* + How the text is positioned relative to its point, defaulting to + ``'center'``. + + *width* + The maximum line length; if non-zero, lines are wrapped at spaces. + + *angle* + How many degrees to rotate the text counter-clockwise about its + positioning point, from ``0.0`` to ``360.0`` (default ``0.0``). + + *underline* + The index of a character to underline, or ``-1`` for none. + + For ``bitmap`` items: + + *bitmap* + The bitmap to display. + + *anchor* + How the bitmap is positioned relative to its point. + + *background*, *foreground* + The colors used for the bitmap's ``0`` and ``1`` pixels; an empty + *background* makes the ``0`` pixels transparent. + Both have *active...* and *disabled...* variants, and *bitmap* has + *activebitmap* and *disabledbitmap* variants. + + For ``image`` items: + + *image* + The Tk image to display, previously created with the image + protocols. + + *anchor* + How the image is positioned relative to its point. + + Both options have *active...* and *disabled...* variants + (*activeimage*, *disabledimage*) used in the active and disabled + states. + + For ``window`` items: + + *window* + The widget to embed; it must be a child of the canvas or of one of + its ancestors, and may not be a top-level window. + + *anchor* + How the window is positioned relative to its point. + + *width*, *height* + The size to assign to the window; if zero (the default), the window + is given its requested size. + + ``oval`` and ``rectangle`` items have no type-specific options; they + use only the standard item options. + + .. note:: + + Tk 8.6 added the *angle* option and Tk 9.0 added the *underline* + option for ``text`` items. + + .. method:: coords(tagOrId) + coords(tagOrId, coordList, /) + coords(tagOrId, /, *coordList) + + Query or modify the coordinates of an item. + With only *tagOrId*, return a list of the floating-point coordinates of + the item given by *tagOrId* (the first matching item if it matches + several). + Given new coordinates, replace the coordinates of that item with them; + like the ``create_*`` methods, the coordinates may be given as separate + numbers, as a single sequence, or as coordinate pairs. + The returned coordinates are always in pixels, regardless of the units + used to specify them; for rectangles, ovals and arcs they are ordered + left, top, right, bottom. + + .. versionchanged:: 3.12 + The arguments are now flattened: the coordinates may be given as + separate arguments, as a single sequence, or grouped in pairs, like + the ``create_*`` methods. + + + .. method:: move(tagOrId, xAmount, yAmount, /) + + Move each of the items given by *tagOrId* in the canvas coordinate space + by adding *xAmount* to every x-coordinate and *yAmount* to every + y-coordinate of the item. + + .. method:: moveto(tagOrId, x='', y='') + + Move the items given by *tagOrId* so that the first coordinate pair (the + upper-left corner of the bounding box) of the lowest matching item is at + position (*x*, *y*). + *x* or *y* may be an empty string, in which case the corresponding + coordinate is unchanged. + All matching items keep their positions relative to each other. + + .. versionadded:: 3.8 + + + .. method:: scale(tagOrId, xOrigin, yOrigin, xScale, yScale, /) + + Rescale the coordinates of all items given by *tagOrId* in canvas + coordinate space. + Each x-coordinate is adjusted so that its distance from *xOrigin* changes + by a factor of *xScale*, and each y-coordinate so that its distance from + *yOrigin* changes by a factor of *yScale* (a factor of ``1.0`` leaves the + coordinate unchanged). + + .. method:: rotate(tagOrId, xOrigin, yOrigin, angle, /) + + Rotate the coordinates of all items given by *tagOrId* in canvas + coordinate space about the origin (*xOrigin*, *yOrigin*) by *angle* + degrees anticlockwise. + Negative values of *angle* rotate clockwise. + + .. versionadded:: next + + .. method:: delete(*tagOrIds) + + Delete each of the items given by the *tagOrIds* arguments. + + .. method:: dchars(tagOrId, first, /) + dchars(tagOrId, first, last, /) + + Delete from each of the items given by *tagOrId* the characters (for text + items) or coordinates (for line and polygon items) in the range from + *first* to *last* inclusive; *last* defaults to *first*. + Items that do not support indexing ignore this operation. + + .. method:: insert(tagOrId, beforeThis, string, /) + + Insert *string* into each of the items given by *tagOrId* just before the + character or coordinate whose index is *beforeThis*. + For line and polygon items *string* must be a valid sequence of + coordinates. + + .. method:: rchars(tagOrId, first, last, string, /) + + Replace the characters (for text items) or coordinates (for line and + polygon items) in the range from *first* to *last* inclusive of each of + the items given by *tagOrId* with *string*. + For line and polygon items *string* must be a valid sequence of + coordinates. + Items that do not support indexing ignore this operation. + + .. versionadded:: next + + .. method:: itemcget(tagOrId, option) + + Return the current value of the configuration option *option* for the + item given by *tagOrId* (the lowest matching item if it matches several). + This is like :meth:`~Misc.cget` but applies to an individual item. + + .. method:: itemconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(tagOrId, cnf=None, **kw) + + Query or modify the configuration options of the items given by + *tagOrId*. + This mirrors :meth:`~Misc.configure`, except that it applies to + individual items rather than to the canvas as a whole. + With no options, it returns a dictionary describing the current options + of the first matching item; otherwise it sets the given options on every + matching item. + The legal options are those accepted by the corresponding ``create_*`` + method. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: type(tagOrId) + + Return the type of the item given by *tagOrId* (the first matching item + if it matches several), such as ``'rectangle'`` or ``'text'``, or + ``None`` if *tagOrId* does not match any item. + + .. method:: gettags(tagOrId, /) + + Return a tuple of the tags associated with the item given by *tagOrId* + (the first matching item in display-list order if it matches several). + Return an empty tuple if no item matches or the item has no tags. + + .. method:: dtag(tagOrId, /) + dtag(tagOrId, tagToDelete, /) + + Remove the tag *tagToDelete* (which defaults to *tagOrId*) from each of + the items given by *tagOrId*. + Items that do not have that tag are unaffected. + + .. method:: addtag(newtag, searchSpec, /, *args) + + Add the tag *newtag* to each item selected by the search specification + *searchSpec* (and any further *args*). + *searchSpec* is one of ``'above'``, ``'all'``, ``'below'``, + ``'closest'``, ``'enclosed'``, ``'overlapping'`` or ``'withtag'``; the + ``addtag_*`` methods below are convenient wrappers that supply each of + these forms. + + .. method:: addtag_above(newtag, tagOrId) + + Add the tag *newtag* to the item just above (after) *tagOrId* in the + display list. + + .. method:: addtag_all(newtag) + + Add the tag *newtag* to all items in the canvas. + + .. method:: addtag_below(newtag, tagOrId) + + Add the tag *newtag* to the item just below (before) *tagOrId* in the + display list. + + .. method:: addtag_closest(newtag, x, y, halo=None, start=None) + + Add the tag *newtag* to the item closest to the point (*x*, *y*). + If *halo* is given, any item within that distance of the point is treated + as overlapping it. + If *start* is given (a tag or id), select the topmost closest item that + lies below *start* in the display list, which can be used to step through + all the closest items. + + .. method:: addtag_enclosed(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= *y2*. + + .. method:: addtag_overlapping(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item that overlaps or is enclosed within + the rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= + *y2*. + + .. method:: addtag_withtag(newtag, tagOrId) + + Add the tag *newtag* to every item given by *tagOrId*. + + .. method:: find(searchSpec, /, *args) + + Return a tuple of the ids of all items selected by the search + specification *searchSpec* (and any further *args*), in stacking order + with the lowest item first. + The search specification has any of the forms accepted by :meth:`addtag`. + The ``find_*`` methods below are more convenient wrappers around it. + + .. method:: find_above(tagOrId) + + Return a tuple containing the id of the item just above *tagOrId* in the + display list. + + .. method:: find_all() + + Return a tuple of the ids of all items in the canvas, in stacking order. + + .. method:: find_below(tagOrId) + + Return a tuple containing the id of the item just below *tagOrId* in the + display list. + + .. method:: find_closest(x, y, halo=None, start=None) + + Return a tuple containing the id of the item closest to the point (*x*, + *y*). + *halo* and *start* are interpreted as for :meth:`addtag_closest`. + + .. method:: find_enclosed(x1, y1, x2, y2) + + Return a tuple of the ids of all items completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_overlapping(x1, y1, x2, y2) + + Return a tuple of the ids of all items that overlap or are enclosed + within the rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_withtag(tagOrId) + + Return a tuple of the ids of all items given by *tagOrId*. + + .. method:: lift(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tkraise(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tag_raise(tagOrId, aboveThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just above the item given by *aboveThis*, or to the top of the display + list if *aboveThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items, whose stacking order is + controlled by :meth:`Misc.tkraise` and :meth:`Misc.lower` instead. + :meth:`lift` and :meth:`tkraise` are aliases of :meth:`!tag_raise`. + + .. method:: lower(tagOrId, belowThis=None, /) + :no-typesetting: + + .. method:: tag_lower(tagOrId, belowThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just below the item given by *belowThis*, or to the bottom of the display + list if *belowThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items. + :meth:`lower` is an alias of :meth:`!tag_lower`. + + .. note:: + + On a :class:`Canvas`, :meth:`tkraise`/:meth:`lift` and :meth:`lower` + restack canvas items, + shadowing the inherited :meth:`Misc.tkraise`/:meth:`Misc.lift` and + :meth:`Misc.lower` methods that restack the widget itself, + which are therefore not available. + + .. method:: tag_bind(tagOrId, sequence=None, func=None, add=None) + + Bind the callback *func* to the event *sequence* for all items given by + *tagOrId*, so that *func* is invoked whenever that event occurs for one + of the items. + This is like :meth:`Widget.bind ` but operates on canvas items + rather than on whole widgets; only mouse, keyboard and virtual events may + be bound. + Mouse events are directed to the current item and keyboard events to the + focus item (see :meth:`focus`). + If *add* is true the new binding is added to any existing bindings for + the same sequence, rather than replacing them. + Return the identifier of the bound function, which can be passed to + :meth:`tag_unbind`. + + .. method:: tag_unbind(tagOrId, sequence, funcid=None) + + Remove for all items given by *tagOrId* the binding for the event + *sequence*. + If *funcid* is given, only that callback (as returned by + :meth:`tag_bind`) is unbound and deregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: bbox(tagOrId, /, *tagOrIds) + + Return a 4-tuple ``(x1, y1, x2, y2)`` giving an approximate bounding box, + in pixels, that encloses all the items given by *tagOrId* and any further + *tagOrIds*. + The result may overestimate the true bounding box by a few pixels. + Return ``None`` if no item matches or the matching items have nothing to + display. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: canvasx(screenx, gridspacing=None) + + Given a window x-coordinate *screenx*, return the canvas x-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: canvasy(screeny, gridspacing=None) + + Given a window y-coordinate *screeny*, return the canvas y-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: focus() + focus(tagOrId, /) + + With *tagOrId*, set the keyboard focus for the canvas to the first item + given by *tagOrId* that supports the insertion cursor; the focus is left + unchanged if no such item exists. + If *tagOrId* is an empty string, reset the focus so that no item has it. + With no argument, return the id of the item that currently has the focus, + or an empty string if none does. + An item only displays the insertion cursor when both it is the focus item + and its canvas has the input focus. + + This shadows the inherited :meth:`!Misc.focus`; + use :meth:`~Misc.focus_set` to focus the widget itself. + + .. method:: icursor(tagOrId, index, /) + + Set the insertion cursor of the items given by *tagOrId* to just before + the character given by *index*. + Items that do not support an insertion cursor are unaffected. + The cursor is only displayed when the item has the focus, but its + position may be set at any time. + + .. method:: index(tagOrId, index, /) + + Return as an integer the numerical index within *tagOrId* corresponding + to *index*, which is a textual description of a position (for text items + an index into the characters, for line and polygon items an index into + the coordinates). + If *tagOrId* matches several items, the first one that supports indexing + is used. + + .. method:: select_adjust(tagOrId, index) + + Adjust the end of the selection in *tagOrId* nearest to *index* so that + it is at *index*, and make the other end the anchor point for future + :meth:`select_to` calls. + If the selection is not currently in *tagOrId*, this behaves like + :meth:`select_to`. + + .. method:: select_clear() + + Clear the selection if it is in this canvas; otherwise do nothing. + + .. method:: select_from(tagOrId, index) + + Set the selection anchor point to just before the character given by + *index* in the item given by *tagOrId*. + This does not change the selection itself; it sets the fixed end for + future :meth:`select_to` calls. + + .. method:: select_item() + + Return the id of the item that holds the selection, or ``None`` if the + selection is not in this canvas. + Unlike :meth:`find` and the ``find_*`` methods, this returns the id as a + string rather than an integer. + + .. method:: select_to(tagOrId, index) + + Set the selection to the characters of *tagOrId* between the selection + anchor point and *index*, inclusive of *index*. + The anchor point is the one set by the most recent :meth:`select_adjust` + or :meth:`select_from` call. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y, gain=10) + + Scroll the canvas by *gain* times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the canvas at high speed through its window. + + .. method:: postscript(cnf={}, **kw) + + Generate a PostScript (Encapsulated PostScript, version 3.0) + representation of part or all of the canvas. + If the *file* or *channel* option is given, the PostScript is written + there and an empty string is returned; otherwise it is returned as a + string. + By default only the area currently visible in the window is generated, so + it is usually necessary either to call :meth:`~Misc.update` first or to + use the *width* and *height* options. + Supported options include *colormap*, *colormode*, *file*, *fontmap*, + *height*, *pageanchor*, *pageheight*, *pagewidth*, *pagex*, *pagey*, + *rotate*, *width*, *x* and *y*. + + +.. class:: Checkbutton(master=None, cnf={}, **kw) + + A :class:`!Checkbutton` widget displays a textual string, bitmap or image + together with a square indicator, and toggles a boolean selection when + pressed. + It has all the behavior of a simple button and, in addition, can be + selected: when selected the indicator is drawn with a check mark and the + associated variable is set to the ``onvalue``, and when deselected the + indicator is drawn empty and the variable is set to the ``offvalue``. + Inherits from :class:`Widget`. + In addition to the standard widget options, a checkbutton accepts the + options documented in the Tk ``checkbutton`` manual page, such as + *variable*, *onvalue*, *offvalue* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the checkbutton with the + mouse: toggle the selection state of the button and invoke the associated + command, if there is one. + Return the result of the command, or an empty string if no command is + associated with the checkbutton. + This is ignored if the checkbutton's state is ``disabled``. + + .. method:: select() + + Select the checkbutton and set the associated variable to its + ``onvalue``. + + .. method:: deselect() + + Deselect the checkbutton and set the associated variable to its + ``offvalue``. + + .. method:: toggle() + + Toggle the selection state of the button, redisplaying it and modifying + its associated variable to reflect the new state. + + .. method:: flash() + + Flash the checkbutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the checkbutton is left in the same normal or + active state as when the method was called. + This is ignored if the checkbutton's state is ``disabled``. + + +.. class:: Entry(master=None, cnf={}, **kw) + + An :class:`!Entry` widget displays a single line of text and lets the user + edit it. + Inherits from :class:`Widget` and :class:`XView`; since entries can hold + strings too long to fit in the window, they support horizontal scrolling + through :meth:`~XView.xview`. + + In addition to the standard widget options, an entry accepts the options + documented in the Tk ``entry`` manual page. + Notable ones are *textvariable* (the name of a variable kept in sync with + the entry's contents), *show* (if set, each character is displayed as the + given character rather than its true value, useful for password entry), + *validate* and *validatecommand* (which together let a callback accept or + reject edits), and *state* (one of ``'normal'``, ``'disabled'`` or + ``'readonly'``). + + Many of the methods below take an *index* argument that selects a character + in the entry's string. + As described in the Tk ``entry`` manual page, *index* may be a number + (counting from 0), ``'insert'`` (the character just after the insertion + cursor), ``'end'`` (just after the last character), ``'anchor'`` (the + selection anchor point), ``'sel.first'`` and ``'sel.last'`` (the ends of the + selection), or ``@x`` (the character covering pixel x-coordinate *x* in the + window). + Out-of-range indices are rounded to the nearest legal value. + + .. method:: delete(first, last=None) + + Delete the characters from index *first* up to but not including index + *last*. + If *last* is omitted, only the single character at *first* is deleted. + + .. method:: get() + + Return the entry's current string. + + .. method:: insert(index, string) + + Insert *string* just before the character given by *index*. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*. + + .. method:: select_adjust(index) + :no-typesetting: + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by + *index*, and adjust that end to be at *index* (including but not going + beyond it); the other end becomes the anchor point for future + :meth:`selection_to` calls. + If there is no selection in the entry, a new one is created between + *index* and the most recent anchor point, inclusive. + :meth:`select_adjust` is an alias of :meth:`!selection_adjust`. + + .. method:: select_clear() + :no-typesetting: + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget the method has no effect. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on an :class:`Entry`. + + .. method:: select_from(index) + :no-typesetting: + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection. + :meth:`select_from` is an alias of :meth:`!selection_from`. + + .. method:: select_present() + :no-typesetting: + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the entry, ``False`` + otherwise. + :meth:`select_present` is an alias of :meth:`!selection_present`. + + .. method:: select_range(start, end) + :no-typesetting: + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + :meth:`select_range` is an alias of :meth:`!selection_range`. + + .. method:: select_to(index) + :no-typesetting: + + .. method:: selection_to(index) + + Set the selection between the anchor point and *index*: if *index* is + before the anchor point, the selection runs from *index* up to but not + including the anchor; if *index* is after it, from the anchor up to but + not including *index*; if they coincide, nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If there is no selection in the entry, a new one is created using the + most recent anchor point. + :meth:`select_to` is an alias of :meth:`!selection_to`. + + .. method:: scan_mark(x) + + Record *x* and the current view in the entry window, for use with later + :meth:`scan_dragto` calls. + Typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Compute the difference between *x* and the *x* given to the last + :meth:`scan_mark` call, and adjust the view left or right by 10 times + that difference. + Typically associated with mouse motion events, to produce the effect of + dragging the entry at high speed through the window. + + .. method:: validate() + + Force an evaluation of the command given by the *validatecommand* option, + independently of the conditions specified by the *validate* option, and + return whether the value is considered valid. + + .. versionadded:: next + + +.. class:: Frame(master=None, cnf={}, **kw) + + A :class:`!Frame` widget is a simple container. + Its primary purpose is to act as a spacer or container for complex window + layouts; its only features are its background and an optional 3-D border to + make the frame appear raised or sunken. + Inherits from :class:`Widget`. + Refer to the Tk ``frame`` manual page for the full list of options. + + +.. class:: Label(master=None, cnf={}, **kw) + + A :class:`!Label` widget displays a non-interactive textual string, bitmap + or image. + The displayed text is set with the *text* option or linked to a variable + through *textvariable*, and an image can be shown using the *image* option. + Text must all be in a single font but may occupy multiple lines, and one + character may be underlined with the *underline* option. + Inherits from :class:`Widget`. + Refer to the Tk ``label`` manual page for the full list of options. + + +.. class:: LabelFrame(master=None, cnf={}, **kw) + + A :class:`!LabelFrame` widget is a container that has the features of a + :class:`Frame` plus the ability to display a label. + The label text is set with the *text* option and positioned with + *labelanchor*, or an arbitrary widget may be used as the label by giving it + as the *labelwidget* option. + Inherits from :class:`Widget`. + Refer to the Tk ``labelframe`` manual page for the full list of options. + + +.. class:: Listbox(master=None, cnf={}, **kw) + + A :class:`!Listbox` widget displays a list of single-line text items, one + per line, of which the user can select one or more. + The way the selection behaves is governed by the *selectmode* option, which + is one of ``browse`` (the default; at most one item, which may be dragged + with the mouse), ``single`` (at most one item), ``multiple`` (any number of + items, toggled individually), or ``extended`` (any number of items, + including discontiguous ranges, selected by clicking and dragging). + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``listbox`` manual page for the full list of options. + + Many of the methods take an *index* argument identifying a particular item. + As described in the Tk ``listbox`` manual page, *index* may be a numeric + index (counting from 0 at the top), ``'active'`` (the item with the location + cursor, set with :meth:`activate`), ``'anchor'`` (the selection anchor, set + with :meth:`selection_anchor`), ``'end'`` (the last item, or for + :meth:`index` and :meth:`insert` the position just after it), or ``@x,y`` + (the item covering pixel coordinates *x*, *y* in the listbox window). + Arguments named *first* and *last* are indices of the same forms. + + .. method:: insert(index, *elements) + + Insert the given *elements* as new items just before the item given by + *index*. + If *index* is ``'end'``, the new items are appended to the end of the + list. + + .. method:: delete(first, last=None) + + Delete the items in the range from *first* to *last* inclusive. + If *last* is omitted, it defaults to *first*, so that a single item is + deleted. + + .. method:: get(first, last=None) + + If *last* is omitted, return the contents of the item given by *first*, + or an empty string if *first* refers to a non-existent item. + If *last* is given, return a tuple of all the items in the range from + *first* to *last* inclusive. + + .. method:: size() + + Return the total number of items in the listbox. + + This shadows the inherited :meth:`!Misc.size`; + use :meth:`~Misc.grid_size` for the grid size. + + .. method:: index(index) + + Return the integer index value corresponding to *index*, or ``None`` if + *index* is out of range. + If *index* is ``'end'``, the result is a count of the number of items in + the listbox (not the index of the last item). + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` describing the bounding box, in + pixels relative to the widget, of the text of the item given by *index*. + Return ``None`` if no part of that item is visible on the screen, or if + *index* refers to a non-existent item; if the item is only partly + visible, the result still gives the full area of the item, including the + parts that are not visible. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: nearest(y) + + Given a y-coordinate within the listbox window, return the index of the + visible item nearest to that y-coordinate. + + .. method:: see(index) + + Adjust the view so that the item given by *index* is visible. + If the item is already visible the method has no effect; if it is near an + edge of the window the listbox scrolls just enough to bring it into view + at that edge, otherwise the listbox scrolls to center the item. + + .. method:: activate(index) + + Set the active item to the one given by *index*. + If *index* is outside the range of items, the closest item is activated + instead. + The active item is drawn as specified by the *activestyle* option when + the widget has the input focus, and its index may be retrieved with the + ``'active'`` index. + + .. method:: curselection() + + Return a tuple containing the numerical indices of all of the items that + are currently selected, or an empty tuple if no items are selected. + + .. method:: select_anchor(index) + :no-typesetting: + + .. method:: selection_anchor(index) + + Set the selection anchor to the item given by *index*. + If *index* refers to a non-existent item, the closest item is used. + The selection anchor is the end of the selection that is fixed while + dragging out a selection with the mouse, and may afterwards be referred + to with the ``'anchor'`` index. + :meth:`select_anchor` is an alias of :meth:`!selection_anchor`. + + .. method:: select_clear(first, last=None) + :no-typesetting: + + .. method:: selection_clear(first, last=None) + + Deselect any of the items in the range from *first* to *last* inclusive + that are selected. + The selection state of items outside this range is not changed. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on a :class:`Listbox`. + + .. method:: select_includes(index) + :no-typesetting: + + .. method:: selection_includes(index) + + Return ``True`` if the item given by *index* is currently selected, + ``False`` otherwise. + :meth:`select_includes` is an alias of :meth:`!selection_includes`. + + .. method:: select_set(first, last=None) + :no-typesetting: + + .. method:: selection_set(first, last=None) + + Select all of the items in the range from *first* to *last* inclusive, + without affecting the selection state of items outside that range. + :meth:`select_set` is an alias of :meth:`!selection_set`. + + .. method:: itemcget(index, option) + + Return the current value of the configuration option *option* for the + item given by *index*. + + .. method:: itemconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the item given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual item rather than to the listbox as a whole. + With no options, it returns a dictionary describing the current options + of the item; otherwise it sets the given options. + The supported item options are *background*, *foreground*, + *selectbackground* and *selectforeground*. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the listbox by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the list at high speed through the window. + + +.. class:: Menu(master=None, cnf={}, **kw) + + A :class:`!Menu` widget displays a column of entries, each of which may be a + command, a checkbutton, a radiobutton, a cascade (which posts an associated + submenu) or a separator. + Menus are used as the menubar of a toplevel window, as pulldown menus posted + from a cascade entry or menubutton, and as popup menus. + Inherits from :class:`Widget`. + + Many of the entry methods take an *index* argument that selects which entry + to operate on. + As described in the Tk ``menu`` manual page, *index* may be a numeric index + (counting from 0 at the top), ``'active'`` (the currently active entry), + ``'end'`` or ``'last'`` (the bottommost entry), ``'none'`` (no entry at all, + written ``{}`` in Tcl), ``@y`` (the entry covering pixel y-coordinate *y* in + the menu window), or a pattern matched against the labels of the entries + from the top down. + + .. method:: add(itemType, cnf={}, **kw) + + Add a new entry to the bottom of the menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'`` and determines the type of the new + entry; the remaining options configure it. + The :meth:`!add_command`, :meth:`!add_cascade`, :meth:`!add_checkbutton`, + :meth:`!add_radiobutton` and :meth:`!add_separator` convenience methods + call this method with the corresponding *itemType*. + + The entry is configured by the following options, although not every + option applies to every entry type (a separator accepts none of them): + + *label* + The text to display in the entry. + + *command* + The function to call when the entry is invoked (command, checkbutton + and radiobutton entries). + + *accelerator* + A string displayed at the right of the entry to advertise an + accelerator keystroke; it does not itself create the binding. + + *underline* + The index of a character in the label to underline for keyboard + traversal. + + *state* + One of ``'normal'``, ``'active'`` or ``'disabled'``. + + *image* + An image to display instead of, or together with, the text label. + + *compound* + Where to show the image relative to the text: ``'none'`` (the + default), ``'text'``, ``'image'``, ``'top'``, ``'bottom'``, ``'left'`` + or ``'right'``. + + *bitmap* + A bitmap to display instead of the text label. + + *font* + The font to use for the text. + + *background*, *foreground* + The entry's background and foreground colors in its normal state + (ignored on macOS). + + *activebackground*, *activeforeground* + The background and foreground colors used when the entry is active + (ignored on macOS). + + *columnbreak* + If true, the entry starts a new column instead of being placed below + the previous entry. + + *hidemargin* + If true, the standard margin around the entry is omitted, which is + useful when a menu is used as a palette. + + *menu* + The submenu posted by a cascade entry; it must be a child of this + menu. + + *variable* + The variable associated with a checkbutton or radiobutton entry. + + *onvalue*, *offvalue* + The values stored in *variable* when a checkbutton entry is selected + or cleared. + + *value* + The value stored in *variable* when a radiobutton entry is selected. + + *indicatoron* + Whether to display the indicator of a checkbutton or radiobutton + entry. + + *selectcolor* + The color of the indicator of a checkbutton or radiobutton entry when + it is selected. + + *selectimage* + The image displayed when a checkbutton or radiobutton entry is + selected and *image* is also given. + + .. method:: add_cascade(cnf={}, **kw) + + Add a new cascade entry to the bottom of the menu. + A cascade entry has an associated submenu, given by its *menu* option, + which must be a child of this menu; posting the entry posts the submenu + next to it. + + .. method:: add_checkbutton(cnf={}, **kw) + + Add a new checkbutton entry to the bottom of the menu. + When invoked, a checkbutton entry toggles between its *onvalue* and + *offvalue*, storing the result in its associated *variable*, and displays + an indicator showing whether it is selected. + + .. method:: add_command(cnf={}, **kw) + + Add a new command entry to the bottom of the menu. + A command entry behaves much like a button: when it is invoked, the + callback given by its *command* option is called. + + .. method:: add_radiobutton(cnf={}, **kw) + + Add a new radiobutton entry to the bottom of the menu. + Radiobutton entries sharing the same *variable* form a group of which + only one may be selected at a time; selecting an entry stores its *value* + in the variable. + + .. method:: add_separator(cnf={}, **kw) + + Add a separator to the bottom of the menu. + A separator is displayed as a horizontal dividing line and cannot be + activated or invoked. + + .. method:: insert(index, itemType, cnf={}, **kw) + + Same as :meth:`add`, except that the new entry is inserted just before + the entry given by *index* instead of being appended to the end of the + menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'``. + The :meth:`!insert_command`, :meth:`!insert_cascade`, + :meth:`!insert_checkbutton`, :meth:`!insert_radiobutton` and + :meth:`!insert_separator` convenience methods call this method with the + corresponding *itemType*. + + .. method:: insert_cascade(index, cnf={}, **kw) + + Insert a new cascade entry before the entry given by *index* (see + :meth:`add_cascade`). + + .. method:: insert_checkbutton(index, cnf={}, **kw) + + Insert a new checkbutton entry before the entry given by *index* (see + :meth:`add_checkbutton`). + + .. method:: insert_command(index, cnf={}, **kw) + + Insert a new command entry before the entry given by *index* (see + :meth:`add_command`). + + .. method:: insert_radiobutton(index, cnf={}, **kw) + + Insert a new radiobutton entry before the entry given by *index* (see + :meth:`add_radiobutton`). + + .. method:: insert_separator(index, cnf={}, **kw) + + Insert a separator before the entry given by *index* (see + :meth:`add_separator`). + + .. method:: delete(index1, index2=None) + + Delete all of the menu entries between *index1* and *index2* inclusive. + If *index2* is omitted, it defaults to *index1*, so that a single entry + is deleted. + Attempts to delete a tear-off entry are ignored; remove it by changing + the *tearoff* option instead. + + .. method:: entrycget(index, option) + + Return the current value of the configuration option *option* for the + entry given by *index*. + + .. method:: entryconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: entryconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the entry given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual entry rather than to the menu as a whole. + With no options, it returns a dictionary describing the current options + of the entry; otherwise it sets the given options. + The supported options are those accepted by :meth:`add` for the entry's + type. + :meth:`entryconfig` is an alias of :meth:`!entryconfigure`. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, or ``None`` if + *index* selects no entry. + + .. method:: type(index) + + Return the type of the entry given by *index*: one of ``'command'``, + ``'cascade'``, ``'checkbutton'``, ``'radiobutton'``, ``'separator'`` or + ``'tearoff'`` (for the tear-off entry). + + .. method:: activate(index) + + Make the entry given by *index* the active entry, redisplaying it with + its active colors, and deactivate any previously active entry. + If *index* selects no entry, or the selected entry is disabled, the menu + ends up with no active entry. + + .. method:: invoke(index) + + Invoke the action of the entry given by *index*, as if it had been + clicked. + Nothing happens if the entry is disabled. + If the entry has a *command* associated with it, the result of that + command is returned; otherwise the result is an empty string. + + .. method:: post(x, y) + + Display the menu on the screen at the root-window coordinates *x* and + *y*, adjusting them if necessary so that the whole menu is visible. + If the *postcommand* option has been specified, it is evaluated before + the menu is posted. + + .. method:: postcascade(index) + + Post the submenu associated with the cascade entry given by *index*, + unposting any previously posted submenu. + This has no effect if *index* does not name a cascade entry or if the + menu itself is not posted. + + .. versionadded:: next + + .. method:: tk_popup(x, y, entry='') + + Post the menu as a popup at the root-window coordinates *x* and *y*. + If *entry* is given, the menu is positioned so that this entry is + displayed under the pointer. + + .. method:: unpost() + + Unmap the menu so that it is no longer displayed, also unposting any + posted lower-level cascaded submenu. + This has no effect on Windows and macOS, which manage the unposting of + menus themselves. + + .. method:: xposition(index) + + Return the x-coordinate, within the menu window, of the leftmost pixel of + the entry given by *index*. + + .. versionadded:: 3.3 + + .. method:: yposition(index) + + Return the y-coordinate, within the menu window, of the topmost pixel of + the entry given by *index*. + + +.. class:: Menubutton(master=None, cnf={}, **kw) + + A :class:`!Menubutton` widget displays a textual string, bitmap or image and + posts an associated :class:`Menu`, given by its *menu* option, when the user + presses it. + Like a :class:`Label` it can show *text*, a *textvariable*, or an *image*, + and the *direction* option controls where the menu appears relative to the + button. + Inherits from :class:`Widget`. + Refer to the Tk ``menubutton`` manual page for the full list of options. + + +.. class:: Message(master=None, cnf={}, **kw) + + A :class:`!Message` widget displays a non-interactive textual string, given + by the *text* option or linked to a variable through *textvariable*. + Unlike a :class:`Label`, it breaks the string into multiple lines in order + to produce a given aspect ratio, choosing line breaks at word boundaries, + and it can justify the text left, centered or right. + Inherits from :class:`Widget`. + Refer to the Tk ``message`` manual page for the full list of options. + + +.. class:: OptionMenu(master, variable, value, *values, **kwargs) + + A helper subclass of :class:`Menubutton` that displays a pop-up menu of + mutually exclusive choices. + *variable* is a :class:`Variable` kept in sync with the selection, *value* + is the initial choice, and *values* are the remaining menu entries. + The keyword argument *command* may be given a callback that is invoked with + the selected value, and the keyword argument *name* sets the Tk widget name. + Other keyword arguments are passed to the underlying :class:`Menubutton` + and may override its default appearance. + + .. method:: destroy() + + Destroy the widget, also cleaning up the associated pop-up menu. + + .. versionchanged:: 3.14 + Added support for the *name* keyword argument. + + .. versionchanged:: next + Other :class:`Menubutton` options can now be passed as keyword arguments. + + + +.. class:: PanedWindow(master=None, cnf={}, **kw) + + A :class:`!PanedWindow` is a geometry-manager widget that arranges any + number of child *panes* in a row (when *orient* is ``'horizontal'``) or a + column (when *orient* is ``'vertical'``). + Each pane holds one widget, and each pair of adjacent panes is separated by + a movable *sash* that the user can drag with the mouse to resize the widgets + on either side of it. + Inherits from :class:`Widget`. + + The *orient* option selects the layout direction, *sashwidth* sets the width + of each sash and *sashrelief* its relief. + When *showhandle* is true a small handle is drawn on each sash that the user + can grab to drag it. + Refer to the Tk ``panedwindow`` manual page for the full list of options. + + .. method:: add(child, **kw) + + Add *child* to the panedwindow as a new pane, placed after any existing + panes. + The keyword arguments specify per-pane management options for *child*; + they may be any of the options accepted by :meth:`paneconfigure`. + + .. method:: forget(child) + :no-typesetting: + + .. method:: remove(child) + + Remove the pane containing *child* from the panedwindow. + All geometry management options for *child* are forgotten. + :meth:`forget` is an alias of :meth:`!remove`. + This shadows the inherited geometry-manager :meth:`!forget`; + use :meth:`~Pack.pack_forget`, :meth:`~Grid.grid_forget` or + :meth:`~Place.place_forget` to remove the widget itself from its manager. + + .. method:: panes() + + Return a tuple of the widgets managed by the panedwindow, one per pane, + in order. + + .. method:: panecget(child, option) + + Return the current value of the management option *option* for the pane + containing *child*. + *option* may be any value allowed by :meth:`paneconfigure`. + + .. method:: paneconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: paneconfigure(tagOrId, cnf=None, **kw) + + Query or modify the management options of the pane containing the widget + *tagOrId*. + With no options, it returns a dictionary describing all of the available + options for the pane; given a single option name as a string, it returns + a description of that one option; otherwise it sets the given options. + The supported options include *after* and *before* (insert the pane after + or before another managed window), *height* and *width* (the outer + dimensions of the window, including any border), *minsize* (the minimum + size in the paned dimension), *padx* and *pady* (extra space to leave on + each side of the window), *sticky* (position or stretch the window within + an oversized pane, using a string of the characters ``n``, ``s``, ``e`` + and ``w``), *hide* (hide the pane while keeping it in the list of panes) + and *stretch* (how extra space is allocated to the pane: one of + ``'always'``, ``'first'``, ``'last'``, ``'middle'`` or ``'never'``). + :meth:`paneconfig` is an alias of :meth:`!paneconfigure`. + + .. method:: identify(x, y) + + Identify the panedwindow component underneath the point given by *x* and + *y*, in window coordinates. + If the point is over a sash or a sash handle, the result is a two-element + tuple containing the index of the sash or handle and a word indicating + whether it is over a sash or a handle, such as ``(0, 'sash')`` or + ``(2, 'handle')``. + If the point is over any other part of the panedwindow, the result is an + empty string. + + .. method:: sash(*args) + + Query or change the position of the sashes in the panedwindow. + This is a thin wrapper around the Tk ``sash`` subcommand; the convenience + methods :meth:`sash_coord`, :meth:`sash_mark` and :meth:`sash_place` + should normally be used instead. + + .. method:: sash_coord(index) + + Return the current x and y coordinate pair for the sash given by *index*, + which must be an integer between 0 and one less than the number of panes + in the panedwindow. + The coordinates returned are those of the top left corner of the region + containing the sash. + + .. method:: sash_mark(index) + + Record the current mouse position for the sash given by *index*, for use + together with later sash-drag operations to move the sash. + + .. method:: sash_place(index, x, y) + + Place the sash given by *index* at the coordinates *x* and *y*. + + .. method:: proxy(*args) + + Query or change the position of the sash proxy, the "ghost" sash shown + while a sash is being dragged with non-opaque resizing. + This is a thin wrapper around the Tk ``proxy`` subcommand; the + convenience methods :meth:`proxy_coord`, :meth:`proxy_forget` and + :meth:`proxy_place` should normally be used instead. + + .. method:: proxy_coord() + + Return a tuple containing the x and y coordinates of the most recent + proxy location. + + .. method:: proxy_forget() + + Remove the proxy from the display. + + .. method:: proxy_place(x, y) + + Place the proxy at the coordinates *x* and *y*. + + +.. class:: Radiobutton(master=None, cnf={}, **kw) + + A :class:`!Radiobutton` widget displays a textual string, bitmap or image + together with a diamond or circular indicator, and selects one choice out of + several. + It has all the behavior of a simple button and, in addition, can be + selected: typically several radiobuttons share a single *variable*, and + selecting one sets that variable to the radiobutton's *value*; each + radiobutton also monitors the variable and automatically selects or + deselects itself when the variable changes. + Inherits from :class:`Widget`. + In addition to the standard widget options, a radiobutton accepts the + options documented in the Tk ``radiobutton`` manual page, such as + *variable*, *value* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the radiobutton with the + mouse: select the button and invoke the associated command, if there is + one. + Return the result of the command, or an empty string if no command is + associated with the radiobutton. + This is ignored if the radiobutton's state is ``disabled``. + + .. method:: select() + + Select the radiobutton and set the associated variable to the value + corresponding to this widget. + + .. method:: deselect() + + Deselect the radiobutton and set the associated variable to an empty + string. + If this radiobutton was not currently selected, this has no effect. + + .. method:: flash() + + Flash the radiobutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the radiobutton is left in the same normal or + active state as when the method was called. + This is ignored if the radiobutton's state is ``disabled``. + + +.. class:: Scale(master=None, cnf={}, **kw) + + A :class:`!Scale` widget lets the user select a numerical value by moving a + slider along a trough. + It can be oriented vertically or horizontally and can optionally display a + label and the current value. + Inherits from :class:`Widget`. + + In addition to the standard widget options, a scale accepts the options + documented in the Tk ``scale`` manual page, such as *from_*, *to*, + *resolution*, *orient*, *tickinterval*, *variable* and *command*. + As elsewhere in :mod:`!tkinter`, the leading ``-`` of the Tk option name is + dropped; *from* is spelled ``from_`` because :keyword:`from` is a Python + keyword. + + With a non-integer *resolution*, see :ref:`numeric values and the locale + `. + + .. method:: get() + + Return the current value of the scale. + The result is an integer if the scale's *resolution* yields whole + numbers, and a float otherwise. + + .. method:: set(value) + + Set the scale to *value*, moving the slider accordingly. + This has no effect if the scale is disabled. + + .. method:: coords(value=None) + + Return a tuple ``(x, y)`` giving the pixel coordinates, relative to the + widget, of the point on the centerline of the trough that corresponds to + *value*. + If *value* is omitted, the scale's current value is used. + + .. method:: identify(x, y) + + Return a string describing the part of the scale at the pixel coordinates + *x*, *y*: ``'slider'``, ``'trough1'`` (the part of the trough above or to + the left of the slider), ``'trough2'`` (below or to the right of the + slider), or an empty string if the point is not over any of these + elements. + + +.. class:: Scrollbar(master=None, cnf={}, **kw) + + A :class:`!Scrollbar` widget displays a slider and two arrows that let the + user scroll an associated widget, such as a :class:`Listbox`, :class:`Text`, + :class:`Canvas` or :class:`Entry`. + It is connected to the scrolled widget by setting that widget's + *xscrollcommand* or *yscrollcommand* option to the scrollbar's :meth:`set` + method, and the scrollbar's *command* option to the scrolled widget's + :meth:`~XView.xview` or :meth:`~YView.yview` method. + Inherits from :class:`Widget`. + + .. method:: get() + + Return the current scrollbar settings as a tuple ``(first, last)`` of two + fractions between 0 and 1, describing the portion of the document that is + currently visible, as last passed to :meth:`set`. + + .. method:: set(first, last) + + Set the scrollbar. + *first* and *last* are fractions between 0 and 1 giving the positions of + the start and end of the visible portion of the associated document. + This method is normally registered as the scrolled widget's + *xscrollcommand* or *yscrollcommand* and called by that widget. + + .. method:: activate(index=None) + + Mark the element *index* (one of ``'arrow1'``, ``'slider'`` or + ``'arrow2'``) as active, displaying it according to the + *activebackground* and *activerelief* options. + If *index* is omitted, return the name of the currently active element, + or ``None`` if no element is active. + + .. versionchanged:: 3.5 + The *index* argument is now optional. + + .. method:: delta(deltax, deltay) + + Return a float indicating the fractional change in the scrollbar setting + that corresponds to moving the slider by *deltax* pixels horizontally + (for horizontal scrollbars) or *deltay* pixels vertically (for vertical + scrollbars). + + .. method:: fraction(x, y) + + Return a float between 0 and 1 indicating where the point at pixel + coordinates *x*, *y* lies in the trough: 0 corresponds to the top or left + of the trough and 1 to the bottom or right. + + .. method:: identify(x, y) + + Return the name of the element under the pixel coordinates *x*, *y* (such + as ``'arrow1'``), or an empty string if the point does not lie in any + element of the scrollbar. + + +.. class:: Spinbox(master=None, cnf={}, **kw) + + A :class:`!Spinbox` widget is an :class:`Entry`-like widget with a pair of + up/down arrow buttons that let the user step through a range of values in + addition to editing the value directly. + The set of values may be a numeric range given by the *from_*, *to* and + *increment* options, or an explicit list of strings given by the *values* + option (which takes precedence over the range). + Each time an arrow is invoked the *command* callback, if any, is called; the + *wrap* option controls whether stepping past either end of the range wraps + around to the other end; the *format* option specifies how numeric values + are formatted; and the *validate* option enables validation of the entered + text. + Inherits from :class:`Widget` and :class:`XView`. + + With a non-integer *increment*, see :ref:`numeric values and the locale + `. + + Many of the methods take an *index* argument identifying a character in the + spinbox's string. + As described in the Tk ``spinbox`` manual page, *index* may be a numeric + index (counting from 0), ``'anchor'`` (the selection anchor point), + ``'end'`` (just after the last character), ``'insert'`` (the character just + after the insertion cursor), ``'sel.first'`` or ``'sel.last'`` (the ends of + the selection), or ``@x`` (the character covering pixel x-coordinate *x* in + the window). + + .. method:: get() + + Return the spinbox's string. + + .. method:: insert(index, s) + + Insert the characters of the string *s* just before the character given + by *index*. + + .. method:: delete(first, last=None) + + Delete one or more characters of the spinbox. + *first* is the index of the first character to delete, and *last* is the + index of the character just after the last one to delete. + If *last* is omitted, a single character at *first* is deleted. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, as a string. + + .. method:: bbox(index) + + Return a tuple of four integers ``(x, y, width, height)`` describing the + bounding box of the character given by *index*. + *x* and *y* are the pixel coordinates of the upper-left corner of the + character relative to the widget, and *width* and *height* are its size + in pixels. + The bounding box may refer to a region outside the visible area of the + window. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: identify(x, y) + + Return the name of the window element at the pixel coordinates *x*, *y*: + one of ``'buttondown'``, ``'buttonup'``, ``'entry'`` or ``'none'``. + + .. method:: invoke(element) + + Invoke the spin button given by *element*, either ``'buttonup'`` or + ``'buttondown'``, triggering the action associated with it. + + .. method:: scan(*args) + + A thin wrapper around the Tk ``scan`` widget subcommand, used to + implement fast dragging of the view: ``scan('mark', x)`` records *x* and + the current view, and ``scan('dragto', x)`` adjusts the view relative to + that mark. + The :meth:`scan_mark` and :meth:`scan_dragto` methods wrap the two forms. + + .. method:: scan_mark(x) + + Record *x* and the current view in the spinbox window, for use with a + later :meth:`scan_dragto` call. + This is typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Adjust the view by 10 times the difference between *x* and the *x* passed + to the last :meth:`scan_mark` call. + This is typically associated with mouse motion events, producing the + effect of dragging the spinbox at high speed through the window. + + .. method:: selection(*args) + + A thin wrapper around the Tk ``selection`` widget subcommand, used to + adjust the selection within the spinbox. + It has several forms depending on the first argument, such as + ``selection('adjust', index)``, ``selection('clear')``, + ``selection('element', ?elem?)``, ``selection('from', index)``, + ``selection('present')``, ``selection('range', start, end)`` and + ``selection('to', index)``. + The :meth:`selection_adjust`, :meth:`selection_clear`, + :meth:`selection_element`, :meth:`selection_from`, + :meth:`selection_present`, :meth:`selection_range` and + :meth:`selection_to` methods wrap these forms. + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by *index* + and adjust that end of the selection to be at *index* (including but not + going beyond *index*). + The other end becomes the anchor point for future :meth:`selection_to` + calls. + If the selection is not currently in the spinbox, a new selection is + created to include the characters between *index* and the most recent + anchor point, inclusive. + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget, the method has no effect. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on a :class:`Spinbox`. + + .. method:: selection_element(element=None) + + Set or get the currently selected element. + If *element* (one of ``'buttonup'``, ``'buttondown'`` or ``'none'``) is + given, that spin button is selected and displayed depressed; otherwise + the name of the currently selected element is returned. + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection itself. + + .. versionadded:: 3.8 + + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the spinbox, + ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + + .. versionadded:: 3.8 + + + .. method:: selection_to(index) + + Set the selection between *index* and the anchor point. + If *index* is before the anchor point, the selection runs from *index* up + to but not including the anchor point; if it is after, the selection runs + from the anchor point up to but not including *index*; if it is the same, + nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If the selection is not in this widget, a new selection is created using + the most recent anchor point. + + .. versionadded:: 3.8 + + .. method:: validate() + + Force an evaluation of the command given by the *validatecommand* option, + independently of the conditions specified by the *validate* option, and + return whether the value is considered valid. + + .. versionadded:: next + + + +.. class:: Text(master=None, cnf={}, **kw) + + A :class:`!Text` widget displays and edits multi-line text. + Portions of the text may be styled with **tags**, particular positions may + be annotated with floating **marks**, and arbitrary images and other widgets + may be embedded in the text. + The widget also provides an unlimited undo/redo mechanism and supports peer + widgets that share the same underlying data. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``text`` manual page for the full list of options. + + Most of the methods take one or more *index* arguments that identify a + position within the text. + As described in the Tk ``text`` manual page, an index is a string consisting + of a base, optionally followed by one or more modifiers. + The base may be ``'line.char'`` (line *line*, character *char*, where lines + are counted from 1 and characters within a line from 0; ``'line.end'`` + refers to the newline ending the line), ``'end'`` (the position just after + the last newline), the name of a mark, ``'tag.first'`` or ``'tag.last'`` + (the first character tagged with *tag*, or the position just after the last + such character), the name of an embedded image or window, or ``@x,y`` (the + character covering pixel coordinates *x*, *y* in the widget). + A modifier such as ``'+5 chars'``, ``'-3 lines'``, ``'linestart'``, + ``'lineend'``, ``'wordstart'`` or ``'wordend'`` adjusts the index relative + to its base; several modifiers may be combined and are applied from left to + right, for example ``'insert wordstart - 1 c'``. + + .. method:: insert(index, chars, *args) + + Insert the string *chars* just before the character at *index* (if + *index* is ``'end'``, just before the final newline). + By default the new text inherits any tags present on both sides of the + insertion point. + If *args* is given, it consists of alternating *tagList*, *chars* values: + the preceding *chars* receives exactly the tags listed (a tag list may be + a single tag name or a sequence of names), overriding the surrounding + tags. + + .. method:: delete(index1, index2=None) + + Delete the range of characters from *index1* up to but not including + *index2*. + If *index2* is omitted, the single character at *index1* is deleted. + The widget always keeps a newline as its last character, so a deletion + that would remove it is adjusted accordingly. + + .. method:: replace(index1, index2, chars, *args) + + Replace the range of characters from *index1* up to but not including + *index2* with *chars*. + This is equivalent to a :meth:`delete` followed by an :meth:`insert` at + *index1*; *args* is interpreted as for :meth:`insert`. + + .. versionadded:: 3.3 + + .. method:: get(index1, index2=None) + + Return the text from *index1* up to but not including *index2* as a + string. + If *index2* is omitted, return the single character at *index1*. + Embedded images and windows are omitted from the result. + + .. method:: index(index) + + Return the position corresponding to *index* in the canonical + ``'line.char'`` form. + + .. method:: compare(index1, op, index2) + + Compare the positions of *index1* and *index2* using the relational + operator *op*, which must be one of ``'<'``, ``'<='``, ``'=='``, + ``'>='``, ``'>'`` or ``'!='``, and return the boolean result. + + .. method:: count(index1, index2, *options, return_ints=False) + + Count the number of items of the requested kinds between *index1* and + *index2*; the count is negative if *index1* is after *index2*. + Each of *options* names a kind of item to count: ``'chars'``, + ``'displaychars'``, ``'displayindices'``, ``'displaylines'``, + ``'indices'``, ``'lines'``, ``'xpixels'`` or ``'ypixels'`` (the default, + used when no option is given, is ``'indices'``). + The pseudo-option ``'update'`` forces any out-of-date layout information + to be recalculated before the following options are evaluated. + When *return_ints* is true and a single counting option is given, return + a plain integer; otherwise return a tuple with one integer per counting + option (or ``None`` if the result is empty). + + .. versionadded:: 3.3 + + .. versionchanged:: 3.13 + Added the *return_ints* parameter. + + + .. method:: see(index) + + Adjust the view so that the character given by *index* is visible. + If it is already visible the method has no effect; if it is a short + distance out of view the widget scrolls just enough to bring it to the + nearest edge, otherwise it scrolls to center *index* in the window. + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` giving the bounding box, in + pixels, of the visible part of the character at *index*, or ``None`` if + that character is not visible on the screen. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: dlineinfo(index) + + Return a tuple ``(x, y, width, height, baseline)`` describing the display + line that contains *index*: the first four values give the bounding box + of the line in pixels and *baseline* gives the offset of the baseline + measured down from the top of the area. + Return ``None`` if that display line is not visible on the screen. + + .. method:: mark_set(markName, index) + + Set the mark named *markName* to the position just before the character + at *index*, creating the mark if it does not already exist. + A mark created this way has right gravity by default. + + .. method:: mark_unset(*markNames) + + Remove each of the marks named in *markNames*. + The special ``insert`` and ``current`` marks may not be removed. + + .. method:: mark_names() + + Return a tuple of the names of all marks currently set in the widget. + + .. method:: mark_gravity(markName, direction=None) + + If *direction* is omitted, return the gravity of mark *markName*, either + ``'left'`` or ``'right'``. + Otherwise set its gravity to *direction*. + The gravity determines on which side of the mark text inserted at the + mark's position appears: a mark with right gravity (the default) stays to + the right of such text. + + .. method:: mark_next(index) + + Return the name of the first mark at or after *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just after that + mark. + + .. method:: mark_previous(index) + + Return the name of the last mark at or before *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just before that + mark. + + .. method:: tag_add(tagName, index1, *args) + + Add the tag *tagName* to the range of characters from *index1* up to but + not including the next index in *args*. + Further pairs of indices may follow in *args* to tag additional ranges; a + trailing single index tags just the character at that index. + + .. method:: tag_remove(tagName, index1, index2=None) + + Remove the tag *tagName* from the characters from *index1* up to but not + including *index2* (or from the single character at *index1* if *index2* + is omitted). + The tag itself continues to exist even if no characters carry it. + + .. method:: tag_delete(*tagNames) + + Delete each of the tags named in *tagNames*, removing them from all + characters and discarding their options and bindings. + + .. method:: tag_config(tagName, cnf=None, **kw) + :no-typesetting: + + .. method:: tag_configure(tagName, cnf=None, **kw) + + Query or modify the configuration options of the tag *tagName*. + This mirrors :meth:`~Misc.configure`, except that it applies to a tag + rather than to the widget as a whole: with no options it returns a + dictionary describing the current options, otherwise it sets the given + options. + Defining a tag this way also gives it a priority higher than any existing + tag. + + The supported tag options, all controlling the appearance of the tagged + text, are: + + *font* + The font to use for the text. + + *foreground* + The color to use for the text. + + *background* + The color to use for the area behind the text. + + *fgstipple*, *bgstipple* + Bitmaps used to stipple the foreground (text) and the background; + only well supported on X11. + + *borderwidth* + The width of the border drawn around the text according to *relief* + (default ``0``). + + *relief* + The 3-D appearance of the text's border: ``'flat'`` (the default), + ``'raised'``, ``'sunken'``, ``'ridge'``, ``'groove'`` or ``'solid'``. + + *offset* + How far the text is raised above (or, if negative, lowered below) the + baseline, for superscripts and subscripts. + + *underline* + Whether to underline the text. + + *underlinefg* + The color of the underline; it defaults to the text color. + + *overstrike* + Whether to draw a line through the middle of the text. + + *overstrikefg* + The color of the overstrike line; it defaults to the text color. + + *elide* + Whether the text is elided (hidden). + + *justify* + How to justify the first character of a display line: ``'left'`` (the + default), ``'right'`` or ``'center'``. + + *wrap* + How to wrap lines that are too long: ``'char'``, ``'word'`` or + ``'none'``. + + *lmargin1*, *lmargin2* + The indentation, in pixels, of the first display line of a logical + line and of the remaining display lines. + + *lmargincolor* + The color of the left margin area. + + *rmargin* + The right-hand margin, in pixels. + + *rmargincolor* + The color of the right margin area. + + *spacing1*, *spacing2*, *spacing3* + Extra space, in pixels, above the first display line of a logical + line, between its display lines, and below its last display line. + + *tabs* + The set of tab stops, in the same form as the widget's *tabs* option. + + *tabstyle* + How tab stops are interpreted: ``'tabular'`` or ``'wordprocessor'``. + + *selectbackground*, *selectforeground* + The background and foreground colors used for the text while it is + selected. + + .. note:: + + Tk 8.6 added the *lmargincolor*, *overstrikefg*, *rmargincolor*, + *selectbackground*, *selectforeground* and *underlinefg* options. + + :meth:`tag_config` is an alias of :meth:`!tag_configure`. + + .. method:: tag_cget(tagName, option) + + Return the current value of the configuration option *option* for the tag + *tagName*. + + .. method:: tag_names(index=None) + + If *index* is omitted, return a tuple of the names of all tags defined in + the widget; otherwise return only the names of the tags applied to the + character at *index*. + The names are ordered from lowest to highest priority. + + .. method:: tag_ranges(tagName) + + Return a tuple of indices describing all ranges of text tagged with + *tagName*. + The result alternates start and end indices, so that elements ``2*i`` and + ``2*i+1`` bound the *i*-th range. + + .. method:: tag_nextrange(tagName, index1, index2=None) + + Search forward from *index1* (up to *index2* if given) for the first + range of characters tagged with *tagName*, and return a two-element tuple + of its start and end indices, or an empty tuple if there is no such + range. + + .. method:: tag_prevrange(tagName, index1, index2=None) + + Search backward from *index1* (down to *index2* if given) for the nearest + preceding range of characters tagged with *tagName*, and return a + two-element tuple of its start and end indices, or an empty tuple if + there is no such range. + + .. method:: tag_raise(tagName, aboveThis=None) + + Raise the priority of tag *tagName* so that it is just above the priority + of *aboveThis*, or to the highest priority of all tags if *aboveThis* is + omitted. + When the display options of overlapping tags conflict, the + higher-priority tag wins. + + .. method:: tag_lower(tagName, belowThis=None) + + Lower the priority of tag *tagName* so that it is just below the priority + of *belowThis*, or to the lowest priority of all tags if *belowThis* is + omitted. + + .. method:: tag_bind(tagName, sequence, func, add=None) + + Bind the event *sequence* on characters tagged with *tagName* to the + callback *func*, so that *func* is invoked when that event occurs over + such a character. + If *add* is true the binding is added alongside any existing bindings for + *sequence*, otherwise it replaces them. + Works like :meth:`~Misc.bind` and returns the identifier of the new + binding. + + .. method:: tag_unbind(tagName, sequence, funcid=None) + + Remove the bindings of the event *sequence* on characters tagged with + *tagName*. + If *funcid* is given, only that binding (as returned by :meth:`tag_bind`) + is removed and its callback is unregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: image_create(index, cnf={}, **kw) + + Embed an image at *index* and return the name assigned to this image + instance, which may then be used as an index or passed to the other + ``image_*`` methods. + The options, given in *cnf* and *kw*, include *image* (the Tk image to + display), *name* (a base name for the instance), *align*, *padx* and + *pady*. + + .. method:: image_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded image at *index*. + + .. method:: image_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded image at + *index*, like :meth:`~Misc.configure` but applied to that image. + + .. method:: image_names() + + Return a tuple of the names of all images embedded in the widget. + + .. note:: + + This shadows the inherited :meth:`Misc.image_names`, + which returns the names of all images in the Tcl interpreter; + that method is not available on a :class:`Text`. + + .. method:: window_create(index, cnf={}, **kw) + + Embed a window (any widget) at *index*. + The options, given in *cnf* and *kw*, include *window* (the widget to + embed), *create* (a callback that creates the widget on demand), *align*, + *stretch*, *padx* and *pady*. + The embedded widget must be a descendant of the text widget's parent. + + .. method:: window_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded window at *index*. + + .. method:: window_config(index, cnf=None, **kw) + :no-typesetting: + + .. method:: window_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded window at + *index*, like :meth:`~Misc.configure` but applied to that window. + + :meth:`window_config` is an alias of :meth:`!window_configure`. + + .. method:: window_names() + + Return a tuple of the names of all windows embedded in the widget. + + .. method:: edit(*args) + + Low-level wrapper around the Tk ``edit`` widget command that controls the + undo/redo mechanism and the modified flag; *args* is the ``edit`` + subcommand and its arguments. + The :meth:`!edit_\*` methods below are thin wrappers around it and are + usually more convenient. + + .. method:: edit_modified(arg=None) + + If *arg* is omitted, return the current state of the modified flag as + ``0`` or ``1``; the flag is set automatically whenever the text is + inserted or deleted. + Otherwise set the flag to the boolean *arg*. + + .. method:: edit_canundo() + + Return ``True`` if there is an edit action on the undo stack that can be + undone, and ``False`` otherwise. + + .. versionadded:: next + + .. method:: edit_canredo() + + Return ``True`` if there is an edit action on the redo stack that can be + reapplied, and ``False`` otherwise. + + .. versionadded:: next + + .. method:: edit_undo() + + Undo the most recent edit action, that is, all the inserts and deletes + recorded on the undo stack since the previous separator, and move it to + the redo stack. + Raises :exc:`TclError` if the undo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_redo() + + Reapply the most recently undone edit action, provided no further edits + have been made since, and move it back to the undo stack. + Raises :exc:`TclError` if the redo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_reset() + + Clear the undo and redo stacks. + + .. method:: edit_separator() + + Push a separator onto the undo stack, marking a boundary between edit + actions for undo and redo. + Has no effect unless the *undo* option is true. + Separators are inserted automatically when the *autoseparators* option is + true. + + .. method:: search(pattern, index, stopindex=None, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None, *, nolinestop=None, strictlimits=None) + + Search for *pattern* starting at *index* and return the index of the + first character of the first match, or an empty string if there is no + match. + Searching stops at *stopindex* if given; otherwise it wraps around the + ends of the text until the starting position is reached again. + The following boolean keyword flags control the search: *forwards* or + *backwards* select the direction (forward is the default); *exact* (the + default) or *regexp* select literal or regular-expression matching; + *nocase* makes the match case-insensitive; *elide* causes hidden text to + be searched as well; *nolinestop* (regexp only) lets ``.`` and ``[^`` + match newlines; and *strictlimits* requires the whole match to lie within + *index* and *stopindex*. + If *count* is a :class:`Variable`, the number of index positions in the + match is stored in it. + + .. versionchanged:: 3.15 + Added the *nolinestop* and *strictlimits* parameters. + + + .. method:: search_all(pattern, index, stopindex=None, *, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None, nolinestop=None, overlap=None, strictlimits=None) + + Like :meth:`search`, but find every match in the searched range and + return a tuple of the starting indices of all matches (empty if there are + none). + By default overlapping matches are not reported; passing a true *overlap* + returns every match that is not wholly contained in another. + If *count* is a :class:`Variable`, it receives a list with one element + per match. + + .. versionadded:: 3.15 + + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the widget by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events, producing the effect of + dragging the text at high speed through the window. + + .. method:: debug(boolean=None) + + If *boolean* is omitted, return whether internal consistency checks of + the B-tree data structure are enabled. + Otherwise enable or disable them. + The setting is shared by all text widgets and may noticeably slow down + widgets holding large amounts of text. + + .. method:: dump(index1, index2=None, command=None, **kw) + + Return the contents of the widget from *index1* up to but not including + *index2* (or just the segment at *index1* if *index2* is omitted), + including text and information about marks, tags, images and windows. + The result is a list of ``(key, value, index)`` triples, where *key* is + one of ``'text'``, ``'mark'``, ``'tagon'``, ``'tagoff'``, ``'image'`` or + ``'window'``. + By default all kinds are reported; passing any of the keyword arguments + *all*, *text*, *mark*, *tag*, *image* or *window* as true restricts the + dump to the selected kinds. + If *command* is given, it is called once per triple with the three values + as arguments and nothing is returned. + + .. method:: peer_create(newPathName, cnf={}, **kw) + + Create a peer text widget with the path name *newPathName* that shares + this widget's underlying data (text, marks, tags, images and the undo + stack). + Changes made through any peer are reflected in all of them. + By default the peer covers the same lines as this widget; standard text + options, including *startline* and *endline*, may be given to override + this. + + .. versionadded:: 3.3 + + .. method:: peer_names() + + Return a tuple of the path names of this widget's peers, not including + the widget itself. + + .. versionadded:: 3.3 + + .. method:: sync(command=None) + + Control the synchronization of the displayed view with the underlying + text, which may lag behind when line heights have not yet been computed + (for example, for lines that have never been displayed). + If *command* is omitted, bring the line metrics up to date immediately by + forcing computation of any outdated line heights, and return once they + are current. + Otherwise schedule *command* to be called, with no arguments, exactly + once as soon as all line heights are up to date; if there are no pending + calculations, it is called immediately. + + .. versionadded:: next + + .. method:: pendingsync() + + Return ``True`` if the line height calculations are not up to date, and + ``False`` otherwise. + The ``<>`` virtual event fires whenever this state + changes, with the *detail* field set to the new value. + + .. versionadded:: next + + .. method:: yview_pickplace(*what) + + Adjust the view so that the location given by *what* is visible. + This is an obsolete equivalent of :meth:`see`, which should be used + instead. + + +Variable classes +^^^^^^^^^^^^^^^^ + +.. class:: Variable(master=None, value=None, name=None) + + The base class for the Tk variable wrappers. + A Tk variable is a value stored in the Tcl interpreter that can be linked to + widgets through their *variable* or *textvariable* options (see + :ref:`coupling-widget-variables`), so that changes propagate both ways: + updating the variable updates every widget bound to it, and a user editing + such a widget updates the variable. + + *master* is the widget whose Tcl interpreter owns the variable; if omitted, + the default root window is used. + *value* is the initial value; if omitted, a type-specific default is used. + *name* is the name of the variable in the Tcl interpreter; if omitted, a + unique name of the form ``'PY_VARnum'`` is generated. + If *name* matches an existing variable and *value* is omitted, the existing + value is retained. + + In most cases you should use one of the typed subclasses below -- + :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar` or + :class:`BooleanVar` -- rather than :class:`!Variable` directly. + + .. versionchanged:: 3.10 + Two variables now compare equal (``==``) only when they have the same + name, are of the same class, and belong to the same Tcl interpreter. + + .. method:: get() + + Return the current value of the variable. + For the base class the value is returned as a string; the typed + subclasses convert it to the appropriate Python type. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*. + :meth:`initialize` is an alias of :meth:`!set`. + + .. versionadded:: 3.3 + The *initialize* spelling. + + .. method:: trace_add(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'array'``, ``'read'``, ``'write'`` or + ``'unset'``, or a list or tuple of such strings. + + When triggered, *callback* is called with three arguments: the name of + the Tcl variable, an index (or an empty string if the variable is not an + element of an array), and the *mode* that triggered the call. + + Return the internal name of the registered callback, which can be passed + to :meth:`trace_remove`. + + .. versionadded:: 3.6 + + .. method:: trace_remove(mode, cbname) + + Remove a trace callback from the variable. + *mode* must match the *mode* that was passed to :meth:`trace_add`, and + *cbname* is the callback name returned by :meth:`trace_add`. + + .. versionadded:: 3.6 + + .. method:: trace_info() + + Return a list of ``(modes, cbname)`` pairs describing all traces + currently set on the variable, where *modes* is a tuple of mode strings + and *cbname* is the internal callback name. + + .. versionadded:: 3.6 + + .. method:: trace(mode, callback) + :no-typesetting: + + .. method:: trace_variable(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'r'``, ``'w'`` or ``'u'``, for read, write + or unset. + Return the internal name of the registered callback. + :meth:`trace` is an alias of :meth:`!trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_add` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + .. method:: trace_vdelete(mode, cbname) + + Remove the trace callback named *cbname* registered for *mode* with + :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_remove` instead. This method wraps a Tcl feature + that was removed in Tcl 9.0. + + .. method:: trace_vinfo() + + Return a list of ``(mode, cbname)`` pairs for all traces set on the + variable with :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_info` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + +.. class:: StringVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a string. + The default value is ``''``. + + .. method:: get() + + Return the value of the variable as a :class:`str`. + + +.. class:: IntVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds an integer. + The default value is ``0``. + + .. method:: get() + + Return the value of the variable as an :class:`int`. + + +.. class:: DoubleVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a float. + The default value is ``0.0``. + + .. method:: get() + + Return the value of the variable as a :class:`float`. + + .. _tkinter-numeric-locale: + + .. note:: + + A floating-point value is always parsed with a period (``.``) as the + decimal separator, but :class:`Spinbox`, :class:`Scale` and + :class:`ttk.Spinbox ` format it according to the + ``LC_NUMERIC`` locale. Under a locale that uses a comma they produce a + value that :meth:`get` cannot read, raising :exc:`TclError`. Set + ``LC_NUMERIC`` to a locale that uses a period (such as ``'C'``) to avoid + this. + + +.. class:: BooleanVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a boolean. + The default value is ``False``. + + .. method:: get() + + Return the value of the variable as a :class:`bool`. + Raise a :exc:`ValueError` if the value cannot be interpreted as a + boolean. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*, converting it to a boolean. + :meth:`initialize` is an alias of :meth:`!set`. + + .. versionadded:: 3.3 + The *initialize* spelling. + + +Image classes +^^^^^^^^^^^^^ + +.. class:: Image(imgtype, name=None, cnf={}, master=None, **kw) + + Base class for Tk images. + *imgtype* is the Tk image type, one of ``'photo'`` or ``'bitmap'``. + An image is a named object that can be displayed by widgets through their + *image* option; deleting all references to the :class:`!Image` object + deletes the underlying Tk image. + Usually you create a :class:`PhotoImage` or :class:`BitmapImage` rather than + an :class:`!Image` directly. + + The image's configuration options are given by *cnf* and *kw* and may be + queried and changed later with the mapping protocol (using ``image[key]``) + or with the :meth:`configure` method. + + .. method:: config(**kw) + :no-typesetting: + + .. method:: configure(**kw) + + Modify one or more configuration options of the image. + The valid options depend on the image type; see :class:`PhotoImage` and + :class:`BitmapImage`. + :meth:`config` is an alias of :meth:`!configure`. + + .. method:: height() + + Return the height of the image, in pixels. + + .. method:: width() + + Return the width of the image, in pixels. + + .. method:: type() + + Return the type of the image, that is the value of *imgtype* with which + it was created (for example ``'photo'`` or ``'bitmap'``). + + +.. class:: PhotoImage(name=None, cnf={}, master=None, **kw) + + A full-color image (the Tk ``photo`` image type), stored internally with a + varying degree of transparency per pixel. + It can read and write GIF, PPM/PGM and (in Tk 8.6 and later) PNG files, read + SVG files (in Tk 9.0 and later), and be drawn in widgets. + Inherits from :class:`Image`. + + The configuration options include *data* (the image contents as a string), + *file* (the name of a file to read the contents from), *format* (the name of + the file format handler), *width* and *height* (the size of the image, used + when building it up piece by piece), *gamma* and *palette*. + + .. method:: blank() + + Blank the image; that is, set the entire image to have no data, so that + it is displayed as transparent and the background of whatever window it + is displayed in shows through. + + .. method:: redither() + + Recalculate the dithered image in each window where it is displayed. + This is useful when the image data was supplied in pieces, in which case + the dithered image may not be exactly correct. + + .. versionadded:: next + + .. method:: cget(option) + + Return the current value of the configuration option *option*. + + .. method:: copy(*, from_coords=None, zoom=None, subsample=None) + + Return a new :class:`PhotoImage` with a copy of this image. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + ``(x1, y1)`` and ``(x2, y2)`` specify diagonally opposite corners of the + rectangle. + If *x2* and *y2* are not specified, they default to the bottom-right + corner of the source image. + The pixels copied include the left and top edges of the rectangle but not + the bottom or right edges. + If *from_coords* is not given, the whole source image is copied. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + .. versionchanged:: 3.13 + Added the *from_coords*, *zoom* and *subsample* parameters. + + + .. method:: copy_replace(sourceImage, *, from_coords=None, to=None, \ + shrink=False, zoom=None, subsample=None, \ + compositingrule=None) + + Copy a region from *sourceImage* (which must be a :class:`PhotoImage`) + into this image, possibly with pixel zooming and/or subsampling. + If no options are specified, the whole of *sourceImage* is copied into + this image, starting at coordinates ``(0, 0)``. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + *to* specifies a rectangular sub-region of the destination image to be + affected. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If *x2* and *y2* are not specified, they default to ``(x1, y1)`` plus the + size of the source region (after subsampling and zooming, if specified). + If *x2* and *y2* are specified, the source region is replicated if + necessary to fill the destination region in a tiled fashion. + + If *shrink* is true, the size of the destination image is reduced, if + necessary, so that the region being copied into is at the bottom-right + corner of the image. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + *compositingrule* specifies how transparent pixels in the source image + are combined with the destination image. + With ``'overlay'`` (the default), the old contents of the destination + image remain visible, as if the source image were printed on a piece of + transparent film and placed over the top of the destination. + With ``'set'``, the old contents of the destination image are discarded + and the source image is used as-is. + + .. versionadded:: 3.13 + + + .. method:: data(format=None, *, from_coords=None, background=None, \ + grayscale=False, metadata=None) + + Return the image data. + + *format* specifies the name of the image file format handler to use. + If it is not given, the data is returned as a tuple (one element per row) + of strings containing space-separated (one element per pixel/column) + colors in ``#RRGGBB`` format. + + *from_coords* specifies a rectangular region of the image to be returned. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region, including ``(x1, y1)`` and excluding ``(x2, y2)``. + If *from_coords* is not given, the whole image is returned. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionadded:: 3.13 + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: get(x, y, *, withalpha=False) + + Return the color of the pixel at coordinates (*x*, *y*) as an + ``(r, g, b)`` tuple of three integers between 0 and 255, representing the + red, green and blue components respectively. + If *withalpha* is true, the returned tuple has a fourth element giving + the alpha (opacity) value of the pixel. + + .. versionchanged:: next + Added the *withalpha* parameter, which requires Tcl/Tk 9.0 or newer. + + .. method:: put(data, to=None, *, format=None, metadata=None) + + Set pixels of the image to the colors given in *data*, which must be a + string or a nested sequence of horizontal rows of pixel colors (for + example ``"{red green} {blue yellow}"``). + + *to* specifies the coordinates of the region of the image into which the + data are copied. + It must be a tuple or a list of 2 or 4 integers ``(x1, y1)`` or + ``(x1, y1, x2, y2)`` giving the top-left corner, and optionally the + bottom-right corner, of the region. + The default position is ``(0, 0)``. + + *format* specifies the format of the image *data*, so that only image + file format handlers whose names begin with it are tried. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionchanged:: next + Added the *format* and *metadata* parameters. + + .. method:: read(filename, format=None, *, from_coords=None, to=None, \ + shrink=False, metadata=None) + + Read image data from the file named *filename* into the image. + + *format* specifies the format of the image data in the file. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + *from_coords* specifies a rectangular sub-region of the image file data + to be copied to the destination image. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image in the file. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole of the image in the file is + read. + + *to* specifies the coordinates of the top-left corner of the region of + the image into which the data are read. + The default is ``(0, 0)``. + + If *shrink* is true, the size of the image is reduced, if necessary, so + that the region into which the file data are read is at the bottom-right + corner of the image. + + .. versionadded:: 3.13 + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: subsample(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` based on this image but using only every + *x*-th pixel in the X direction and every *y*-th pixel in the Y + direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + .. method:: transparency_get(x, y) + + Return ``True`` if the pixel at coordinates (*x*, *y*) is fully + transparent, ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: transparency_set(x, y, boolean) + + Make the pixel at coordinates (*x*, *y*) fully transparent if *boolean* + is true, fully opaque otherwise. + + .. versionadded:: 3.8 + + + .. method:: write(filename, format=None, from_coords=None, *, \ + background=None, grayscale=False, metadata=None) + + Write image data from the image to the file named *filename*. + + *format* specifies the name of the image file format handler to use. + If it is not given, the format is guessed from the file extension. + + *from_coords* specifies a rectangular region of the image to be written. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole image is written. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionchanged:: 3.13 + Added the *background* and *grayscale* parameters. + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: zoom(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` with this image magnified by a factor of + *x* in the X direction and *y* in the Y direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + +.. class:: BitmapImage(name=None, cnf={}, master=None, **kw) + + A two-color image (the Tk ``bitmap`` image type) created from an X11 bitmap. + Each pixel displays a foreground color, a background color, or nothing + (producing a transparent effect). + Inherits from :class:`Image`. + + The configuration options are *data* or *file* (the source bitmap, given as + a string in X11 bitmap format or as the name of a file in that format), + *maskdata* or *maskfile* (the mask bitmap, in the same forms), and + *foreground* and *background* (the two colors). + For pixels where the mask is zero the image displays nothing; for other + pixels it displays the foreground color where the source is one and the + background color where the source is zero. + If *background* is set to an empty string, the background pixels are + transparent. + + :class:`!BitmapImage` has no methods of its own beyond those inherited from + :class:`Image`. + + +Other classes +^^^^^^^^^^^^^ + +.. class:: Event() + + A container for the attributes of an event passed to a callback bound with + :meth:`Misc.bind`. + An :class:`!Event` instance has the following attributes, each corresponding + to a field of the underlying Tk event; depending on the event type, some + attributes may be set to the string ``'??'`` to indicate that they are not + meaningful. + See :ref:`bindings-and-events`. + + .. attribute:: serial + + The serial number of the event. + + .. attribute:: num + + The mouse button that was pressed or released (for button events). + + .. attribute:: focus + + Whether the window has the focus (for ``Enter`` and ``Leave`` events). + + .. attribute:: height + width + + The new height and width of the window (for ``Configure`` and ``Expose`` + events). + + .. attribute:: keycode + + The keycode of the key that was pressed or released. + + .. attribute:: state + + The state of the event, as a number (for most events) or a string (for + ``Visibility`` events). + + .. attribute:: time + + The timestamp of the event, in milliseconds. + + .. attribute:: x + y + + The pointer position relative to the widget, in pixels. + + .. attribute:: x_root + y_root + + The pointer position relative to the top-left corner of the screen, in + pixels. + + .. attribute:: char + + The character typed, as a string (for key events). + + .. attribute:: send_event + + ``True`` if the event was sent by another application. + + .. attribute:: keysym + + The symbolic name of the key that was pressed or released. + + .. attribute:: keysym_num + + The numeric value of :attr:`keysym`. + + .. attribute:: type + + The :class:`EventType` of the event. + + .. attribute:: widget + + The widget on which the event occurred. + + .. attribute:: delta + + The amount the mouse wheel was rotated (for ``MouseWheel`` events). + + .. attribute:: user_data + + The data string of a virtual event, as passed to the *data* option of + :meth:`Misc.event_generate`. + It is ``'??'`` for non-virtual events. + + .. versionadded:: 3.15 + + .. attribute:: detail + + A fixed detail string for ``Enter``, ``Leave``, ``FocusIn``, ``FocusOut`` + and ``ConfigureRequest`` events (see the Tcl/Tk documentation). + It is ``'??'`` for other events. + + .. versionadded:: 3.15 + + +.. class:: EventType(*values) + + An :class:`enum.StrEnum` enumerating the Tk event types, used as the value + of :attr:`Event.type`. + Its members include, among others, ``KeyPress``, ``KeyRelease``, + ``ButtonPress``, ``ButtonRelease``, ``Motion``, ``Enter``, ``Leave``, + ``FocusIn``, ``FocusOut``, ``Configure``, ``Map``, ``Unmap``, ``Expose``, + ``Destroy`` and ``MouseWheel``. + + .. versionadded:: 3.6 + + + +.. class:: CallWrapper(func, subst, widget) + + Internal helper that wraps a Python callback so that it can be invoked from + Tcl. + *func* is the Python function, *subst* is an optional function that + pre-processes the Tcl arguments, and *widget* is the widget used for error + reporting. + Instances are created automatically by :meth:`Misc.register`; this class is + not normally used directly. + + +Module-level functions +^^^^^^^^^^^^^^^^^^^^^^ + +.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) + + The :func:`Tcl` function is a factory function which creates an object much + like that created by the :class:`Tk` class, except that it does not + initialize the Tk subsystem. + This is most often useful when driving the Tcl interpreter in an environment + where one doesn't want to create extraneous toplevel windows, or where one + cannot (such as Unix/Linux systems without an X server). + An object created by the :func:`Tcl` object can have a Toplevel window + created (and the Tk subsystem initialized) by calling its :meth:`~Tk.loadtk` + method. + +.. function:: NoDefaultRoot() + + Inhibit the creation of an implicit default root window. + Afterwards :mod:`!tkinter` no longer creates a shared default root + automatically, and operations that rely on one --- such as constructing a + widget without an explicit *master* --- raise a :exc:`RuntimeError`. + Call this early in larger applications to make the root window explicit. + +.. function:: mainloop(n=0) + + Run the Tk main event loop on the default root window until all windows are + destroyed. + Equivalent to calling :meth:`Misc.mainloop` on the default root. + +.. function:: getboolean(s) + + Convert the Tcl boolean string *s* (one of ``'1'``, ``'true'``, ``'yes'``, + ``'on'`` and similar, or their false counterparts) to a Python + :class:`bool`. + Raise :exc:`TclError` for an invalid value. + +.. function:: getdouble(s) + + Convert *s* to a floating-point number. + This is the built-in :class:`float`. + +.. function:: getint(s) + + Convert *s* to an integer. + This is the built-in :class:`int`. + +.. function:: image_names() + + Return the names of all existing images in the default root's interpreter. + +.. function:: image_types() + + Return the available image types (such as ``'photo'`` and ``'bitmap'``) in + the default root's interpreter. + + +.. _tkinter-file-handlers: + +File handlers +^^^^^^^^^^^^^ + +Tk allows you to register and unregister a callback function which will be +called from the Tk mainloop when I/O is possible on a file descriptor. +Only one handler may be registered per file descriptor. Example code:: + + import tkinter + widget = tkinter.Tk() + mask = tkinter.READABLE | tkinter.WRITABLE + widget.tk.createfilehandler(file, mask, callback) + ... + widget.tk.deletefilehandler(file) + +This feature is not available on Windows. + +Since you don't know how many bytes are available for reading, you may not +want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` +:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, +since these will insist on reading a predefined number of bytes. +For sockets, the :meth:`~socket.socket.recv` or +:meth:`~socket.socket.recvfrom` methods will work fine; for other files, +use raw reads or ``os.read(file.fileno(), maxbytecount)``. + + +.. method:: Widget.tk.createfilehandler(file, mask, func) + + Registers the file handler callback function *func*. The *file* argument + may either be an object with a :meth:`~io.IOBase.fileno` method (such as + a file or socket object), or an integer file descriptor. The *mask* + argument is an ORed combination of any of the three constants below. + The callback is called as follows:: + + callback(file, mask) + + +.. method:: Widget.tk.deletefilehandler(file) + + Unregisters a file handler. + + +.. data:: READABLE + WRITABLE + EXCEPTION + + Constants used in the *mask* arguments. + + +Constants +^^^^^^^^^ + +The following symbolic constants are available in both the :mod:`!tkinter` +and :mod:`!tkinter.constants` namespaces. + +.. data:: TRUE + YES + ON + + Truthy values, all equal to the integer ``1``. + +.. data:: FALSE + NO + OFF + + Falsy values, all equal to the integer ``0``. + +.. data:: N + S + E + W + NE + NW + SE + SW + NS + EW + NSEW + CENTER + + Compass directions (``'n'``, ``'s'``, ``'e'``, ``'w'`` and the diagonals and + edges) plus ``CENTER`` (``'center'``), used as values for the *anchor* and + *sticky* options and by methods such as :meth:`Misc.grid_anchor`. + +.. data:: LEFT + RIGHT + TOP + BOTTOM + + Sides for the *side* option of the packer (see :meth:`Pack.pack_configure`). + +.. data:: X + Y + BOTH + NONE + + Values for the *fill* option of the packer: ``'x'``, ``'y'``, ``'both'`` or + ``'none'``. + +.. data:: RAISED + SUNKEN + FLAT + RIDGE + GROOVE + SOLID + + Values for the *relief* option, which controls a widget's 3-D border. + +.. data:: HORIZONTAL + VERTICAL + + Values for the *orient* option of widgets such as :class:`Scale`, + :class:`Scrollbar` and :class:`PanedWindow`. + +.. data:: CHAR + WORD + + Values for the *wrap* option of the :class:`Text` widget, selecting line + wrapping on character or word boundaries. + +.. data:: BASELINE + + The text-alignment value ``'baseline'``. + +.. data:: INSIDE + OUTSIDE + + Values for the *bordermode* option of the placer (see + :meth:`Place.place_configure`). + +.. data:: INSERT + CURRENT + END + ANCHOR + SEL + SEL_FIRST + SEL_LAST + + Symbolic indices used by the :class:`Text`, :class:`Entry`, :class:`Listbox` + and :class:`Canvas` widgets, such as ``'insert'`` (the insertion cursor), + ``'current'``, ``'end'``, ``'anchor'`` and the bounds of the selection + (``'sel.first'`` and ``'sel.last'``). + +.. data:: ALL + + The special tag ``'all'``, which matches every item of a :class:`Canvas` or + every character of a :class:`Text` (for example ``canvas.delete(ALL)``). + +.. data:: NORMAL + DISABLED + ACTIVE + HIDDEN + + Values for the *state* option of various widgets and items. + +.. data:: CASCADE + CHECKBUTTON + COMMAND + RADIOBUTTON + SEPARATOR + + Menu entry types, used as the *itemType* argument of :meth:`Menu.add` and + :meth:`Menu.insert`. + +.. data:: SINGLE + BROWSE + MULTIPLE + EXTENDED + + Values for the *selectmode* option of the :class:`Listbox` widget. + +.. data:: PIESLICE + CHORD + ARC + + Values for the *style* option of :class:`Canvas` arc items. + +.. data:: BUTT + PROJECTING + ROUND + BEVEL + MITER + + Values for the *capstyle* (``'butt'``, ``'projecting'``, ``'round'``) and + *joinstyle* (``'round'``, ``'bevel'``, ``'miter'``) options of + :class:`Canvas` line items. + +.. data:: FIRST + LAST + + Values for the *arrow* option of :class:`Canvas` line items, indicating + which ends have arrowheads. + +.. data:: MOVETO + SCROLL + + The first argument passed by a :class:`Scrollbar` to the :meth:`XView.xview` + or :meth:`YView.yview` method of the scrolled widget. + +.. data:: UNITS + PAGES + + Values for the *what* argument of :meth:`XView.xview_scroll` and + :meth:`YView.yview_scroll`. + +.. data:: UNDERLINE + NUMERIC + DOTBOX + + Other option values: ``'underline'``, ``'numeric'`` and ``'dotbox'``. diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index eb30b9c3eacc1b6..30aef8748edb72b 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,4 +1,4 @@ -:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +:mod:`!tkinter.scrolledtext` --- Scrolled text widget ===================================================== .. module:: tkinter.scrolledtext @@ -13,10 +13,12 @@ implements a basic text widget which has a vertical scroll bar configured to do the "right thing." Using the :class:`ScrolledText` class is a lot easier than setting up a text widget and scroll bar directly. -The text widget and scrollbar are packed together in a :class:`Frame`, and the -methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired -from the :class:`Frame` object. This allows the :class:`ScrolledText` widget to -be used directly to achieve most normal geometry management behavior. +The text widget and scrollbar are packed together in a :class:`~tkinter.Frame`, +and the methods of the :class:`~tkinter.Pack`, :class:`~tkinter.Grid` and +:class:`~tkinter.Place` geometry managers are acquired from the +:class:`~tkinter.Frame` object. +This allows the :class:`ScrolledText` widget to be used directly to achieve +most normal geometry management behavior. Should more specific control be necessary, the following attributes are available: diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index e1383e189a31a2b..ef7bbd130fb8883 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -19,6 +19,8 @@ The basic idea for :mod:`!tkinter.ttk` is to separate, to the extent possible, the code implementing a widget's behavior from the code implementing its appearance. +.. versionadded:: 3.1 + .. seealso:: @@ -44,12 +46,12 @@ That code causes several :mod:`!tkinter.ttk` widgets (:class:`Button`, :class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to automatically replace the Tk widgets. -This has the direct benefit of using the new widgets which gives a better -look and feel across platforms; however, the replacement widgets are not -completely compatible. The main difference is that widget options such as -"fg", "bg" and others related to widget styling are no -longer present in Ttk widgets. Instead, use the :class:`ttk.Style` class -for improved styling effects. +This has the direct benefit of using the new widgets which gives a better look +and feel across platforms; however, the replacement widgets are not completely +compatible. +The main difference is that widget options such as "fg", "bg" and others +related to widget styling are no longer present in Ttk widgets. +Instead, use the :class:`ttk.Style {%- endblock -%} {% block body %}

{{ docstitle|e }}

@@ -21,63 +50,76 @@

{{ docstitle|e }}

{% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

{% trans %}Documentation sections:{% endtrans %}

- - -
-

{% trans %}What's new in Python {{ version }}?{% endtrans %}
- {% trans whatsnew_index=pathto("whatsnew/index") %}Or all "What's new" documents since Python 2.0{% endtrans %}

-

{% trans %}Tutorial{% endtrans %}
- {% trans %}Start here: a tour of Python's syntax and features{% endtrans %}

-

{% trans %}Library reference{% endtrans %}
- {% trans %}Standard library and builtins{% endtrans %}

-

{% trans %}Language reference{% endtrans %}
- {% trans %}Syntax and language elements{% endtrans %}

-

{% trans %}Python setup and usage{% endtrans %}
- {% trans %}How to install, configure, and use Python{% endtrans %}

-

{% trans %}Python HOWTOs{% endtrans %}
- {% trans %}In-depth topic manuals{% endtrans %}

- 		-

{% trans %}Installing Python modules{% endtrans %}
- {% trans %}Third-party modules and PyPI.org{% endtrans %}

-

{% trans %}Distributing Python modules{% endtrans %}
- {% trans %}Publishing modules for use by other people{% endtrans %}

-

{% trans %}Extending and embedding{% endtrans %}
- {% trans %}For C/C++ programmers{% endtrans %}

-

{% trans %}Python's C API{% endtrans %}
- {% trans %}C API reference{% endtrans %}

-

{% trans %}FAQs{% endtrans %}
- {% trans %}Frequently asked questions (with answers!){% endtrans %}

-

{% trans %}Deprecations{% endtrans %}
- {% trans %}Deprecated functionality{% endtrans %}

-
+
+ + +
+ +

{% trans %}Other resources:{% endtrans %}

+
+ + +

{% trans %}Indices, glossary, and search:{% endtrans %}

- - -
-

{% trans %}Global module index{% endtrans %}
- {% trans %}All modules and libraries{% endtrans %}

-

{% trans %}General index{% endtrans %}
- {% trans %}All functions, classes, and terms{% endtrans %}

-

{% trans %}Glossary{% endtrans %}
- {% trans %}Terms explained{% endtrans %}

- 		-

{% trans %}Search page{% endtrans %}
- {% trans %}Search this documentation{% endtrans %}

-

{% trans %}Complete table of contents{% endtrans %}
- {% trans %}Lists all sections and subsections{% endtrans %}

-
+
+ + +

{% trans %}Project information:{% endtrans %}

- - -
-

{% trans %}Reporting issues{% endtrans %}

-

{% trans %}Contributing to docs{% endtrans %}

-

{% trans %}Download the documentation{% endtrans %}

- 		-

{% trans %}History and license of Python{% endtrans %}

-

{% trans %}Copyright{% endtrans %}

-

{% trans %}About the documentation{% endtrans %}

-
+
+ + +
{% endblock %} diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 72cac1c1e909d35..45a2ca21801437f 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -16,7 +16,7 @@ Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.15 + python3.16 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.15 - Python 3.15 (default, May 7 2025, 15:46:04) + $ python3.16 + Python 3.16 (default, May 7 2026, 19:03:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index e7c641044740504..dec2008add1bf50 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python315' + 'C:\\Python316' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 6c68ba010813794..cf3a44850b0b8f2 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -278,7 +278,7 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python315/lib/weakref.py", line 46, in __getitem__ + File "C:/python316/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index f362e1943b666f7..6b6b8a768e3f58d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -88,6 +88,11 @@ For example: '~/envs/tutorial-env/lib/python3.5/site-packages'] >>> +Note that the activated virtual environment does not alter the ``PYTHONPATH`` variable in any way. +This may lead to unexpected results if the path includes references to code which is incompatible with +the Python version the virtual environment is using. The best practice is to ``unset PYTHONPATH`` +in bash or the equivalent for the shell you are using. + To deactivate a virtual environment, type:: deactivate diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 7cbc03f5f1281ec..677fbbae3f4219a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -50,8 +50,8 @@ additional methods of invocation: * When called with ``-c command``, it executes the Python statement(s) given as *command*. Here *command* may contain multiple statements separated by newlines. -* When called with ``-m module-name``, the given module is located on the - Python module path and executed as a script. +* When called with ``-m module-name``, the given module is located using the standard + import mechanism and executed as a script. In non-interactive mode, the entire input is parsed before it is executed. @@ -78,8 +78,8 @@ source. .. option:: -m - Search :data:`sys.path` for the named module and execute its contents as - the :mod:`__main__` module. + Locate the module using the standard import mechanism and execute its contents + as the :mod:`__main__` module. Since the argument is a *module* name, you must not give a file extension (``.py``). The module name should be a valid absolute Python module name, but @@ -662,7 +662,7 @@ Miscellaneous options initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. - .. versionchanged:: next + .. versionchanged:: 3.15 Accept also ``module:func`` entry point format. .. versionadded:: 3.13 @@ -705,10 +705,9 @@ Miscellaneous options .. versionadded:: 3.14 - * :samp:`-X lazy_imports={all,none,normal}` controls lazy import behavior. - ``all`` makes all imports lazy by default, ``none`` disables lazy imports - entirely (even explicit ``lazy`` statements become eager), and ``normal`` - (the default) respects the ``lazy`` keyword in source code. + * :samp:`-X lazy_imports={all,normal}` controls lazy import behavior. + ``all`` makes all imports lazy by default, and ``normal`` (the default) + respects the ``lazy`` keyword in source code. See also :envvar:`PYTHON_LAZY_IMPORTS`. .. versionadded:: 3.15 @@ -1154,9 +1153,6 @@ conflict. 'replace', respectively. Otherwise, the new defaults 'utf-8' and 'surrogatepass' are used. - This may also be enabled at runtime with - :func:`sys._enablelegacywindowsfsencoding`. - .. availability:: Windows. .. versionadded:: 3.6 @@ -1416,10 +1412,9 @@ conflict. .. envvar:: PYTHON_LAZY_IMPORTS - Controls lazy import behavior. Accepts three values: ``all`` makes all - imports lazy by default, ``none`` disables lazy imports entirely (even - explicit ``lazy`` statements become eager), and ``normal`` (the default) - respects the ``lazy`` keyword in source code. + Controls lazy import behavior. Accepts two values: ``all`` makes all + imports lazy by default, and ``normal`` (the default) respects the + ``lazy`` keyword in source code. See also the :option:`-X lazy_imports <-X>` command-line option. @@ -1462,7 +1457,7 @@ Debug-mode variables Needs Python configured with the :option:`--with-pydebug` build option. - .. versionchanged:: next + .. versionchanged:: 3.15 Accept also ``module:func`` entry point format. .. versionadded:: 3.13 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 086f6bfa22ad4ac..22a10db976c4fc6 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -112,7 +112,6 @@ Dependencies to build optional modules are: .. [1] If *libmpdec* is not available, the :mod:`decimal` module will use a pure-Python implementation. - See :option:`--with-system-libmpdec` for details. .. [2] See :option:`--with-readline` for choosing the backend for the :mod:`readline` module. .. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. @@ -474,6 +473,15 @@ General Options .. versionadded:: 3.15 +.. option:: --with-build-details-suffix=[yes|SUFFIX] + + Rename ``build-details.json`` to permit multiple co-located Python + installs. If a custom ``SUFFIX`` is supplied it is used verbatim, + otherwise one will be generated from the ``MULTIARCH`` tag with + ``-free-threading`` and ``-debug``, as appropriate. + + .. versionadded:: 3.16 + C compiler options ------------------ @@ -553,11 +561,6 @@ Options for third-party dependencies C compiler and linker flags for ``libmpdec``, used by :mod:`decimal` module, overriding ``pkg-config``. - .. note:: - - These environment variables have no effect unless - :option:`--with-system-libmpdec` is specified. - .. option:: LIBLZMA_CFLAGS .. option:: LIBLZMA_LIBS @@ -784,10 +787,19 @@ also be used to improve performance. Disable frame pointers, which are enabled by default (see :pep:`831`). - By default, the build appends ``-fno-omit-frame-pointer`` (and - ``-mno-omit-leaf-frame-pointer`` when the compiler supports it) to - ``BASECFLAGS`` so profilers, debuggers, and system tracing tools - (``perf``, ``eBPF``, ``dtrace``, ``gdb``) can walk the C call stack + By default, the build appends flags to generate frame or backchain + pointers to ``BASECFLAGS``: + + - ``-fno-omit-frame-pointer`` and/or ``-mno-omit-leaf-frame-pointer`` + are added when the compiler supports them. + - ``-marm`` and/or ``-mno-thumb`` is added on 32-bit ARM when supported, + - on s390x platforms, when supported, ``-mbackchain`` is added *instead*. + of the above frame pointer flags. + - on ppc64le platforms, no compiler flags is needed since the power ABI + requires that compilers maintain a back chain by default. + + Frame pointers enable profilers, debuggers, and system tracing tools + (``perf``, ``eBPF``, ``dtrace``, ``gdb``) to walk the C call stack without DWARF metadata. The flags propagate to third-party C extensions through :mod:`sysconfig`. On compilers that do not understand them, the build silently skips them. @@ -1043,7 +1055,7 @@ Linker options The default (when ``-enable-shared`` is used) is to link the Python interpreter against the built shared library. - .. versionadded:: next + .. versionadded:: 3.15 Libraries options @@ -1058,29 +1070,6 @@ Libraries options Build the :mod:`!pyexpat` module using an installed ``expat`` library (default is no). -.. option:: --with-system-libmpdec - - Build the ``_decimal`` extension module using an installed ``mpdecimal`` - library, see the :mod:`decimal` module (default is yes). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.13 - Default to using the installed ``mpdecimal`` library. - - .. versionchanged:: 3.15 - - A bundled copy of the library will no longer be selected - implicitly if an installed ``mpdecimal`` library is not found. - In Python 3.15 only, it can still be selected explicitly using - ``--with-system-libmpdec=no`` or ``--without-system-libmpdec``. - - .. deprecated-removed:: 3.13 3.16 - A copy of the ``mpdecimal`` library sources will no longer be distributed - with Python 3.16. - - .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. - .. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index eea1e2f64a468d0..5b715d4614dad8f 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -455,6 +455,12 @@ customization. Python runtimes, or false to prevent it. By default, true. + * - ``shebang_templates`` + - (none) + - Mapping from shebang line template to alternative command, such as + ``py -V:`` or a substitute string. + See :ref:`pymanager-shebang` for more details. + * - ``log_level`` - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` - Set the default level of output (0-50). @@ -568,6 +574,30 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. +Since version 26.3 of the Python install manager, custom shebang templates may +be added to your configuration file. Add the ``shebang_templates`` object with +one member for each template (the string to match) and the command to use when +the template is matched. Most commands should be ``py -V:`` (or ``pyw``) to +launch one of your installed runtimes. The ``py -3.`` form is also +allowed, as is a plain ``py`` to launch the default. No other arguments are +supported. + +.. code:: json5 + + { + "shebang_templates": { + "/usr/bin/python": "py", + "/usr/bin/my_custom_python": "py -V:MyCustomPython/3" + } + } + +If the substitute command is not ``py`` or ``pyw``, it will be written back into +the shebang and regular handling continues. If launching arbitrary executables +is permitted, then providing a full path will allow you to redirect from Python +to any executable. The template should match either the entire line (ignoring +leading and trailing whitespace), or up to the first space in the shebang line. + + .. note:: The behaviour of shebangs in the Python install manager is subtly different diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 03e612fb651e71b..4b92ba82c991e44 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -745,7 +745,7 @@ The contextlib module The new :mod:`contextlib` module provides some functions and a decorator that are useful for writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the :meth:`~object.__enter__` method, and the value yielded will be the method's return diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 1215601a09d681c..e1f72b6c4a6c92c 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -431,7 +431,7 @@ The contextlib module The :mod:`contextlib` module provides some functions and a decorator that are useful when writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`~contextlib.contextmanager`, and lets you write +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the :meth:`~object.__enter__` method, and the value yielded will @@ -1557,8 +1557,9 @@ Some smaller changes made to the core Python language are: .. Revision 57619 -* Properties now have three attributes, :attr:`getter`, :attr:`setter` - and :attr:`deleter`, that are decorators providing useful shortcuts +* Properties now have three decorators, :deco:`~property.getter`, + :deco:`~property.setter` and :deco:`~property.deleter`, that are + decorators providing useful shortcuts for adding a getter, setter or deleter function to an existing property. You would use them like this:: diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 7296296d144803d..de90c37a38d38ff 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1267,7 +1267,7 @@ changes, or look through the Subversion logs for all the details. uploads thanks to an added *rest* parameter (patch by Pablo Mouzo; :issue:`6845`.) -* New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` +* New class decorator: :deco:`~functools.total_ordering` in the :mod:`functools` module takes a class that defines an :meth:`~object.__eq__` method and one of :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, and generates the missing comparison methods. Since the diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 8a78dbd90382ed7..390b82d5a1fdc1b 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -996,7 +996,7 @@ dataclasses __slots__ ~~~~~~~~~ -Added ``slots`` parameter in :func:`dataclasses.dataclass` decorator. +Added ``slots`` parameter in :deco:`dataclasses.dataclass` decorator. (Contributed by Yurii Karabas in :issue:`42269`) Keyword-only fields @@ -1446,8 +1446,8 @@ as a positional-only argument. types ----- -Reintroduce the :data:`types.EllipsisType`, :data:`types.NoneType` -and :data:`types.NotImplementedType` classes, providing a new set +Reintroduce the :class:`types.EllipsisType`, :class:`types.NoneType` +and :class:`types.NotImplementedType` classes, providing a new set of types readily interpretable by type checkers. (Contributed by Bas van Beek in :issue:`41810`.) @@ -1485,9 +1485,9 @@ is a :class:`typing.TypedDict`. Subclasses of ``typing.Protocol`` which only have data variables declared will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`~typing.runtime_checkable`. Previously, these checks +are decorated with :deco:`~typing.runtime_checkable`. Previously, these checks passed silently. Users should decorate their -subclasses with the :func:`!runtime_checkable` decorator +subclasses with the :deco:`!runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`.) @@ -1928,7 +1928,7 @@ Changes in the Python API (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley in :issue:`42392`.) -* The :data:`types.FunctionType` constructor now inherits the current builtins +* The :class:`types.FunctionType` constructor now inherits the current builtins if the *globals* dictionary has no ``"__builtins__"`` key, rather than using ``{"None": None}`` as builtins: same behavior as :func:`eval` and :func:`exec` functions. Defining a function with ``def function(...): ...`` diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index a095d887352127a..7b3475882063635 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -388,7 +388,7 @@ Kumar Srinivasan and Graham Bleaney.) PEP 681: Data class transforms ------------------------------ -:data:`~typing.dataclass_transform` may be used to +:deco:`~typing.dataclass_transform` may be used to decorate a class, metaclass, or a function that is itself a decorator. The presence of ``@dataclass_transform()`` tells a static type checker that the decorated object performs runtime "magic" that transforms a class, @@ -686,8 +686,8 @@ enum * Added the :func:`~enum.member` and :func:`~enum.nonmember` decorators, to ensure the decorated object is/is not converted to an enum member. -* Added the :func:`~enum.property` decorator, - which works like :func:`property` except for enums. +* Added the :deco:`~enum.property` decorator, + which works like :deco:`property` except for enums. Use this instead of :func:`types.DynamicClassAttribute`. * Added the :func:`~enum.global_enum` enum decorator, @@ -740,7 +740,7 @@ fractions functools --------- -* :func:`functools.singledispatch` now supports :class:`types.UnionType` +* :deco:`functools.singledispatch` now supports :class:`types.UnionType` and :class:`typing.Union` as annotations to the dispatch argument.:: >>> from functools import singledispatch @@ -1164,7 +1164,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. type checker errors related to highly dynamic class, such as mocks. (Contributed by Shantanu Jain in :gh:`91154`.) -* The :func:`typing.final` decorator now sets the ``__final__`` attributed on +* The :deco:`typing.final` decorator now sets the ``__final__`` attributed on the decorated object. (Contributed by Jelle Zijlstra in :gh:`90500`.) @@ -1197,7 +1197,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. :data:`~typing.ClassVar` annotations. (Contributed by Gregory Beauregard in :gh:`90711`.) -* :func:`typing.no_type_check` no longer modifies external classes and functions. +* :deco:`typing.no_type_check` no longer modifies external classes and functions. It also now correctly marks classmethods as not to be type checked. (Contributed by Nikita Sobolev in :gh:`90729`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index df6cc98eaf1c90a..9e48b3ab7393c87 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -146,7 +146,7 @@ New typing features: * :ref:`PEP 692 `, using :class:`~typing.TypedDict` to annotate :term:`**kwargs ` -* :ref:`PEP 698 `, :func:`typing.override` decorator +* :ref:`PEP 698 `, :deco:`typing.override` decorator Important deprecations, removals or restrictions: @@ -511,7 +511,7 @@ See :pep:`692` for more details. PEP 698: Override Decorator for Static Typing --------------------------------------------- -A new decorator :func:`typing.override` has been added to the :mod:`typing` +A new decorator :deco:`typing.override` has been added to the :mod:`typing` module. It indicates to type checkers that the method is intended to override a method in a superclass. This allows type checkers to catch mistakes where a method that is intended to override something in a base class @@ -1021,7 +1021,7 @@ typing ``__orig_bases__`` attribute. (Contributed by Adrian Garcia Badaracco in :gh:`103699`.) -* Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. +* Add ``frozen_default`` parameter to :deco:`typing.dataclass_transform`. (Contributed by Erik De Bonte in :gh:`99957`.) unicodedata @@ -1367,6 +1367,8 @@ Deprecated .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst .. _whatsnew312-removed: @@ -1747,7 +1749,7 @@ Changes in the Python API around process-global resources, which are best managed from the main interpreter. (Contributed by Donghee Na in :gh:`99127`.) -* The undocumented locking behavior of :func:`~functools.cached_property` +* The undocumented locking behavior of :deco:`~functools.cached_property` is removed, because it locked across all instances of the class, leading to high lock contention. This means that a cached property getter function could now run more than once for a single instance, if two threads race. For most simple diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index f4489bfa1b74e85..d8845fe921ededa 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1958,14 +1958,14 @@ New Deprecations * :mod:`sys`: - * Deprecate the :func:`~sys._enablelegacywindowsfsencoding` function, + * Deprecate the :func:`!_enablelegacywindowsfsencoding` function, to be removed in Python 3.16. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. (Contributed by Inada Naoki in :gh:`73427`.) * :mod:`tarfile`: - * Deprecate the undocumented and unused :attr:`!TarFile.tarfile` attribute, + * Deprecate the undocumented and unused :attr:`!TarInfo.tarfile` attribute, to be removed in Python 3.16. (Contributed in :gh:`115256`.) @@ -1994,7 +1994,7 @@ New Deprecations use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) - * Deprecate the :func:`!typing.no_type_check_decorator` decorator function, + * Deprecate the :deco:`!typing.no_type_check_decorator` decorator function, to be removed in Python 3.15. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. @@ -2031,6 +2031,8 @@ New Deprecations .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst CPython Bytecode Changes @@ -2597,7 +2599,7 @@ Build Changes .. _mimalloc library: https://github.com/microsoft/mimalloc/ -* The :file:`configure` option :option:`--with-system-libmpdec` +* The :file:`configure` option :option:`!--with-system-libmpdec` now defaults to ``yes``. The bundled copy of ``libmpdec`` will be removed in Python 3.16. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 0bb8858aea16fef..cd0d8b7cb006fee 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -2718,7 +2718,7 @@ New deprecations (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) * :mod:`symtable`: - Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest, + Deprecate :meth:`!symtable.Class.get_methods` due to the lack of interest, scheduled for removal in Python 3.16. (Contributed by Bénédikt Tran in :gh:`119698`.) @@ -2748,6 +2748,8 @@ New deprecations .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 2e408f258e28052..db6903da9f63444 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -75,23 +75,29 @@ Summary -- Release highlights profiling tools ` * :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler ` +* :pep:`831`: :ref:`Frame pointers are enabled by default for improved + system-level observability ` * :pep:`798`: :ref:`Unpacking in comprehensions ` * :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding ` -* :pep:`728`: ``TypedDict`` with typed extra items +* :pep:`829`: :ref:`Package startup configuration files ` +* :pep:`728`: :ref:`TypedDict with typed extra items ` * :pep:`747`: :ref:`Annotating type forms with TypeForm ` * :pep:`800`: Disjoint bases in the type system * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object ` -* :pep:`803`: :ref:`Stable ABI for Free-Threaded Builds ` -* :pep:`831`: :ref:`Frame pointers everywhere ` +* :pep:`803`, :pep:`820 <820>`, :pep:`793 <793>`: + :ref:`Stable ABI for free-threaded builds ` and + related C API +* :pep:`788`: :ref:`Protection against finalization in the C API ` * :ref:`The JIT compiler has been significantly upgraded ` -* :ref:`Improved error messages ` * :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter ` -* :pep:`829`: Package Startup Configuration Files +* :ref:`Improved error messages ` +* :ref:`More color ` + New features ============ @@ -151,11 +157,10 @@ making it straightforward to diagnose and debug the failure. For cases where you want to enable lazy loading globally without modifying source code, Python provides the :option:`-X lazy_imports <-X>` command-line option and the :envvar:`PYTHON_LAZY_IMPORTS` environment variable. Both -accept three values: ``all`` makes all imports lazy by default, ``none`` -disables lazy imports entirely (even explicit ``lazy`` statements become -eager), and ``normal`` (the default) respects the ``lazy`` keyword in source -code. The :func:`sys.set_lazy_imports` and :func:`sys.get_lazy_imports` -functions allow changing and querying this mode at runtime. +accept two values: ``all`` makes all imports lazy by default, and ``normal`` +(the default) respects the ``lazy`` keyword in source code. The +:func:`sys.set_lazy_imports` and :func:`sys.get_lazy_imports` functions allow +changing and querying this mode at runtime. For more selective control, :func:`sys.set_lazy_imports_filter` accepts a callable that determines whether a specific module should be loaded lazily. @@ -178,7 +183,7 @@ eager: import myapp.slow_module # lazy (matches filter) import json # eager (does not match filter) -The proxy type itself is available as :data:`types.LazyImportType` for code +The proxy type itself is available as :class:`types.LazyImportType` for code that needs to detect lazy imports programmatically. There are some restrictions on where the ``lazy`` keyword can be used. Lazy @@ -264,6 +269,8 @@ name. (PEP by Tal Einat; contributed by Jelle Zijlstra in :gh:`148829`.) +.. seealso:: :pep:`661` for further details. + .. _whatsnew315-profiling-package: @@ -376,9 +383,42 @@ available output formats, profiling modes, and configuration options. (Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) +.. _whatsnew315-frame-pointers: + +:pep:`831`: Frame pointers enabled by default +--------------------------------------------- + +CPython is now built with frame pointers by default on platforms that support +them. This uses the compiler flags ``-fno-omit-frame-pointer`` and +``-mno-omit-leaf-frame-pointer``, making native stack unwinding faster and +more reliable for system profilers, debuggers, crash analysis tools, and +eBPF-based observability tools. + +The flags are exposed through :mod:`sysconfig`, so extension modules built by +tools that consume Python's build configuration inherit frame pointers by +default. This propagation is intentional: mixed Python/native profiling needs +an unbroken frame-pointer chain through the interpreter, extension modules, +embedding applications, and native libraries. + +.. important:: + + Third-party build backends and native build systems should preserve these + flags when they consume Python's :mod:`sysconfig` values. Build systems + that compile C, C++, Rust, or other native code without inheriting Python's + compiler flags should enable equivalent frame-pointer flags themselves. A + single native component built without frame pointers can break stack + unwinding for the whole Python process. + +(Contributed by Pablo Galindo Salgado and Savannah Ostrowski in +:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, +Savannah Ostrowski, and Diego Russo.) + +.. seealso:: :pep:`831` for further details. + + .. _whatsnew315-unpacking-in-comprehensions: -:pep:`798`: Unpacking in Comprehensions +:pep:`798`: Unpacking in comprehensions --------------------------------------- List, set, and dictionary comprehensions, as well as generator expressions, now @@ -416,10 +456,46 @@ agen() for x in a)``. (Contributed by Adam Hartz in :gh:`143055`.) +.. _whatsnew315-startup-files: + +:pep:`829`: Package startup configuration files +----------------------------------------------- + +Loaded by the :mod:`site` module when ``-S`` is not given, :ref:`.pth files ` +can contain lines that both extend :data:`sys.path` and execute arbitrary code +when the line starts with ``import`` (followed by a space or tab). The latter +functionality can be problematic, since it is difficult to know exactly what +gets executed when Python starts up. + +As a step towards improving the ability to audit pre-start executable code, +Python 3.15 introduces :ref:`.start files ` which contain entry point +specifications of the form ``pkg.mod:callable`` where ``pkg.mod`` is the +import path to the given callable. When Python starts up, the callable is +located and called with no arguments. + +``import`` lines in :file:`.pth` files are silently deprecated. When a +matching :file:`.start` file is found, ``import`` lines in :file:`.pth` files +are ignored. There is no change to :data:`sys.path` extension lines in +:file:`.pth` files. + +The :mod:`site` module also provides :class:`site.StartupState` to batch +startup processing for multiple site directories, ensuring all static path +extensions are applied before any startup code is executed. :func:`site.main` +uses an instance of this class implicitly to batch process all startup +configuration files during normal interpreter startup. Callers needing the +same batching behavior can build a :class:`~site.StartupState` directly and +drive it with :meth:`~site.StartupState.addsitedir`, +:meth:`~site.StartupState.addusersitepackages`, and +:meth:`~site.StartupState.addsitepackages`, then call +:meth:`~site.StartupState.process` once at the end of the batch. + +(Contributed by Barry Warsaw in :gh:`148641` and :gh:`150228`.) + + .. _whatsnew315-abi3t: -:pep:`803`: ``abi3t`` -- Stable ABI for Free-Threaded Builds ------------------------------------------------------------- +:pep:`803` -- Stable ABI for free-threaded builds +------------------------------------------------- C extensions that target the :ref:`Stable ABI ` can now be compiled for the new *Stable ABI for Free-Threaded Builds* (also known @@ -434,7 +510,8 @@ specifically: part of the instance struct; and - Switching from a ``PyInit_`` function to a new export hook, :c:func:`PyModExport_* `, introduced for this - purpose in :pep:`793`. + purpose in :pep:`793`, with a new :c:type:`PySlot` structure + introduced in :pep:`820`. Note that Stable ABI does not offer all the functionality that CPython has to offer. @@ -451,6 +528,44 @@ If not using a build tool -- or when writing such a tool -- you can select ``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed in :ref:`abi3-compiling`. +A practical :ref:`migration guide ` for switching to +``abi3t`` is available. + +.. seealso:: :pep:`803` for further details. + + +.. _whatsnew315-c-api-interpreter-finalization: + +:pep:`788`: Protecting the C API from interpreter finalization +-------------------------------------------------------------- + +In the C API, :term:`interpreter finalization ` can be +problematic for many extensions, because :term:`attaching ` a thread state will permanently hang the thread, resulting in deadlocks +and other spurious issues. Additionally, it has historically been impossible +to safely check whether an interpreter is alive before using it, leading to crashes +when a thread concurrently deletes an interpreter while another thread is +trying to attach to it. + +There are now several new suites of APIs to circumvent these problems: + +* :ref:`Interpreter guards `, which prevent an interpreter + from finalizing. +* :ref:`Interpreter views `, which allow thread-safe access + to an interpreter that may be concurrently finalizing or deleted. +* :ref:`New APIs ` to automatically attach and detach + thread states that come with built-in protection against finalization. + +In addition, APIs in the ``PyGILState`` family (most notably +:c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`) have been +:term:`soft deprecated`. There is **no** plan to remove them, and existing +code will continue to work, but there will be no new ``PyGILState`` APIs +in future versions of Python. + +.. seealso:: :pep:`788` for further details. + +(Contributed by Peter Bierma in :gh:`149101`.) + .. _whatsnew315-improved-error-messages: @@ -492,6 +607,47 @@ Improved error messages ^^^^^^^^^^^^^^ AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? +* When an :exc:`AttributeError` on a builtin type has no close match via + Levenshtein distance, the error message now checks a static table of common + method names from other languages (JavaScript, Java, Ruby, C#) and suggests + the Python equivalent: + + .. doctest:: + + >>> [1, 2, 3].push(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'list' object has no attribute 'push'. Did you mean '.append'? + + >>> 'hello'.toUpperCase() # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'str' object has no attribute 'toUpperCase'. Did you mean '.upper'? + + When the Python equivalent is a language construct rather than a method, + the hint describes the construct directly: + + .. doctest:: + + >>> {}.put("a", 1) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'dict' object has no attribute 'put'. Use d[k] = v. + + When a mutable method is called on an immutable type, the hint suggests + the mutable counterpart: + + .. doctest:: + + >>> (1, 2, 3).append(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'tuple' object has no attribute 'append'. Did you mean to use a 'list' object? + + These hints also work for subclasses of builtin types. + + (Contributed by Matt Van Horn in :gh:`146406`.) + * The interpreter now tries to provide a suggestion when :func:`delattr` fails due to a missing attribute. When an attribute name that closely resembles an existing attribute is used, @@ -545,10 +701,38 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) + .. _whatsnew315-color-interpreter-help: + +* The interpreter help (such as ``python --help``) is now in color. + This can be controlled by :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148766`.) + + .. _whatsnew315-color-exceptions: + * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) + .. _whatsnew315-more-color: + +* More color in + :ref:`argparse `, + :ref:`ast `, + :ref:`calendar `, + :ref:`difflib `, + :ref:`http.server `, + :ref:`pickletools `, + :ref:`PyREPL tab completion `, + :ref:`python --help `, + :ref:`sqlite3 `, + :ref:`timeit `, + :ref:`tokenize `, + :ref:`unraisable exceptions ` and + :term:`stdlib` (ast, compileall, doctest, gzip, inspect, json.tool, pdb, + profiling.sampling, random, regrtest, sqlite3, timeit, tokenize, trace, + unittest, uuid, zipapp, zipfile) CLI help. + * The :meth:`~object.__repr__` of :class:`ImportError` and :class:`ModuleNotFoundError` now shows "name" and "path" as ``name=`` and ``path=`` if they were given as keyword arguments at construction time. @@ -623,7 +807,7 @@ Other language changes n = buffer.find(b'\n') data = bytes(buffer[:n + 1]) del buffer[:n + 1] - assert data == b'abc' + assert data == b'abc\n' assert buffer == bytearray(b'def') - .. code:: python @@ -671,8 +855,8 @@ Other language changes (Contributed by James Hilton-Balfe in :gh:`128335`.) * The class :class:`memoryview` now supports the :c:expr:`float complex` and - :c:expr:`double complex` C types: formatting characters ``'F'``/``'Zf'`` - and ``'D'``/``'Zd'`` respectively. + :c:expr:`double complex` C types: formatting characters ``'Zf'`` and ``'Zd'`` + respectively. (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) * Allow the *count* argument of :meth:`bytes.replace` to be a keyword. @@ -690,6 +874,19 @@ Other language changes (Contributed by Gregory P. Smith in :gh:`83065`.) +Default interactive shell +========================= + +.. _whatsnew315-pyrepl-completion: + +* Tab completions are now colored by object kind, based on + :pypi:`fancycompleter`. + Set :envvar:`PYTHON_BASIC_COMPLETER` to fall back to :mod:`rlcompleter`. + Color can also be controlled by :ref:`environment variables + `. + (Contributed by Antonio Cuni and Pablo Galindo in :gh:`130472`.) + + New modules =========== @@ -707,7 +904,7 @@ Improved modules argparse -------- -* The :class:`~argparse.BooleanOptionalAction` action supports now single-dash +* The :class:`~argparse.BooleanOptionalAction` action now supports single-dash long options and alternate prefix characters. (Contributed by Serhiy Storchaka in :gh:`138525`.) @@ -715,16 +912,22 @@ argparse default to ``True``. This enables suggestions for mistyped arguments by default. (Contributed by Jakob Schluse in :gh:`140450`.) -* Added backtick markup support in description and epilog text to highlight - inline code when color output is enabled. + .. _whatsnew315-color-argparse: + +* Added backtick markup support in :class:`~argparse.ArgumentParser` description + and epilog text to highlight inline code when color output is enabled. (Contributed by Savannah Ostrowski in :gh:`142390`.) +* Extended backtick markup to argument ``help`` text and added support for + double backticks (RST inline-literal style). + (Contributed by Hugo van Kemenade in :gh:`149375`.) + array ----- * Support the :c:expr:`float complex` and :c:expr:`double complex` C types: - formatting characters ``'F'``/``'Zf'`` and ``'D'``/``'Zd'`` respectively. + formatting characters ``'Zf'`` and ``'Zd'`` respectively. (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) * Support half-floats (16-bit IEEE 754 binary interchange format): formatting @@ -739,6 +942,8 @@ array ast --- + .. _whatsnew315-color-ast: + * Add *color* parameter to :func:`~ast.dump`. If ``True``, the returned string is syntax highlighted using ANSI escape sequences. @@ -836,6 +1041,8 @@ binascii calendar -------- + .. _whatsnew315-color-calendar: + * :mod:`calendar`'s :ref:`command-line ` text output has more color. This can be controlled with :ref:`environment variables `. @@ -899,7 +1106,7 @@ dbm --- * Added new :meth:`!reorganize` methods to :mod:`dbm.dumb` and :mod:`dbm.sqlite3` - which allow to recover unused free space previously occupied by deleted entries. + to recover unused free space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) @@ -919,15 +1126,6 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) -faulthandler ------------- - -* Added the *max_threads* parameter in :func:`faulthandler.enable`, - :func:`faulthandler.dump_traceback`, :func:`faulthandler.dump_traceback_later`, - and :func:`faulthandler.register`. - (Contributed by Eric Froemling in :gh:`149085`.) - - email ----- @@ -938,6 +1136,15 @@ email (Contributed by R David Murray and Mike Edmunds in :gh:`122540`.) +faulthandler +------------ + +* Added the *max_threads* parameter in :func:`faulthandler.enable`, + :func:`faulthandler.dump_traceback`, :func:`faulthandler.dump_traceback_later`, + and :func:`faulthandler.register`. + (Contributed by Eric Froemling in :gh:`149085`.) + + functools --------- @@ -950,6 +1157,17 @@ functools (Contributed by Bartosz Sławecki in :gh:`143535`.) +gc +-- + +* Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later and Python 3.15. + + hashlib ------- @@ -974,6 +1192,8 @@ http.client http.server ----------- + .. _whatsnew315-color-http.server: + * The logging of :mod:`~http.server.BaseHTTPRequestHandler`, as used by the :ref:`command-line interface `, is colored by default. @@ -997,6 +1217,19 @@ http.server (Contributed by Anton I. Sipos in :gh:`135057`.) +importlib.metadata +------------------ + +* Previously, when accessing a distribution metadata directory not + containing a metadata file, ``metadata()`` and ``Distribution.metadata()`` + would return an empty ``PackageMetadata`` object as if the file + was present but empty. Now, a ``MetadataNotFound`` exception is raised. + See `importlib_metadata#493 `_ + for background and rationale and and :gh:`143387` for rationale on the + compatibility concerns. + (Contributed by Jason R. Coombs.) + + inspect ------- @@ -1004,6 +1237,7 @@ inspect for :func:`~inspect.getdoc`. (Contributed by Serhiy Storchaka in :gh:`132686`.) + json ---- @@ -1045,7 +1279,7 @@ mimetypes (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and John Franey in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, - :gh:`145718` and :gh:`144213`.) + :gh:`145718`, :gh:`145918`, and :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. @@ -1073,6 +1307,10 @@ os glibc versions 2.28 and later. (Contributed by Jeffrey Bosboom and Victor Stinner in :gh:`83714`.) +* :func:`os.makedirs` function now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories. This can be used to match + the behavior from Python 3.6 and earlier by passing ``parent_mode=mode``. + (Contributed by Zackery Spytz and Gregory P. Smith in :gh:`86533`.) os.path ------- @@ -1104,6 +1342,8 @@ pickle pickletools ----------- + .. _whatsnew315-color-pickletools: + * The output of the :mod:`pickletools` command-line interface is colored by default. This can be controlled with :ref:`environment variables `. @@ -1173,6 +1413,8 @@ sqlite3 * SQL keyword completion on . (Contributed by Long Tan in :gh:`133393`.) + .. _whatsnew315-color-sqlite3: + * Prompts, error messages, and help text are now colored. This is enabled by default, see :ref:`using-on-controlling-color` for details. @@ -1296,7 +1538,7 @@ tarfile (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` and :cve:`2025-4435`.) * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` - now replace slashes by backslashes in symlink targets on Windows to prevent + now replace slashes with backslashes in symlink targets on Windows to prevent creation of corrupted links. (Contributed by Christoph Walcher in :gh:`57911`.) @@ -1314,10 +1556,13 @@ threading timeit ------ + .. _whatsnew315-color-timeit: + * The output of the :mod:`timeit` command-line interface is colored by default. This can be controlled with :ref:`environment variables `. (Contributed by Hugo van Kemenade in :gh:`146609`.) + * The command-line interface now colorizes error tracebacks by default. This can be controlled with :ref:`environment variables `. @@ -1357,6 +1602,8 @@ tkinter tokenize -------- + .. _whatsnew315-color-tokenize: + * The output of the :mod:`tokenize` :ref:`command-line interface ` is colored by default. This can be controlled with :ref:`environment variables `. @@ -1413,7 +1660,7 @@ types ----- * Expose the write-through :func:`locals` proxy type - as :data:`types.FrameLocalsProxyType`. + as :class:`types.FrameLocalsProxyType`. This represents the type of the :attr:`frame.f_locals` attribute, as described in :pep:`667`. @@ -1442,13 +1689,22 @@ typing (Contributed by Jelle Zijlstra in :gh:`145033`.) +.. _whatsnew315-typeddict: + +* :pep:`728`: Add support in :class:`~typing.TypedDict` for the *closed* + and *extra_items* class arguments. A closed :class:`~typing.TypedDict` + does not allow extra keys beyond those specified in the class body, while + a :class:`~typing.TypedDict` with ``extra_items`` allows arbitrary extra + items where the values are of the specified type. (Contributed by Angela + Liss in :gh:`137840`.) + * Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. (Contributed by Nikita Sobolev in :gh:`137191`.) * Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` - as it was incorrectly inferred in runtime before. + as it was incorrectly inferred at runtime before. (Contributed by Nikita Sobolev in :gh:`137191`.) * :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class @@ -1511,8 +1767,8 @@ urllib.parse :func:`~urllib.parse.urlparse` and :func:`~urllib.parse.urldefrag` functions. Add the *keep_empty* parameter to :func:`~urllib.parse.urlunsplit` and :func:`~urllib.parse.urlunparse` functions. - This allows to distinguish between empty and not defined URI components - and preserve empty components. + This allows distinguishing between empty and undefined URI components + and preserving empty components. (Contributed by Serhiy Storchaka in :gh:`67041`.) @@ -1520,7 +1776,7 @@ venv ---- * On POSIX platforms, platlib directories will be created if needed when - creating virtual environments, instead of using ``lib64 -> lib`` symlink. + creating virtual environments, instead of using a ``lib64 -> lib`` symlink. This means purelib and platlib of virtual environments no longer share the same ``lib`` directory on platforms where :data:`sys.platlibdir` is not equal to ``lib``. @@ -1558,6 +1814,32 @@ wave (Contributed by Lionel Koenig and Michiel W. Beijen in :gh:`60729`.) +webbrowser +---------- + +* On macOS, the new :class:`!webbrowser.MacOS` class opens URLs via + :program:`/usr/bin/open` instead of constructing and executing AppleScript + via :program:`osascript`. The default browser is detected from the + LaunchServices preferences file using :mod:`plistlib`, with + :class:`!com.apple.Safari` as the fallback on fresh installations. + For non-HTTP(S) URLs, :program:`open -b ` is used to route the + URL through a browser rather than the OS file handler, preventing + file injection attacks. + (Contributed by Jeff Lyon in :gh:`137586`.) + + +xml +--- + +* Add the :func:`xml.is_valid_name` function to check + whether a string can be used as an element or attribute name in XML. + (Contributed by Serhiy Storchaka in :gh:`139489`.) + +* Add the :func:`xml.is_valid_text` function, which allows to check + whether a string can be used in the XML document. + (Contributed by Serhiy Storchaka in :gh:`139489`.) + + xml.parsers.expat ----------------- @@ -1648,6 +1930,7 @@ The major upgrades to the JIT are: * New tracing frontend * Basic register allocation in the JIT * More JIT optimizations +* GDB and GNU ``backtrace()`` unwinding support * Better machine code generation .. rubric:: LLVM 21 build-time dependency @@ -1693,12 +1976,23 @@ reduces the cost of most operations in Python. Reiden Ong, Noam Cohen, Tomas Roun, PuQing, Cajetan Rodrigues, and Sacul in :gh:`134584`.) By tracking unique references to objects, the JIT optimizer can now eliminate -reference count updates and perform inplace operations on ints and floats. +reference count updates and perform in-place operations on ints and floats. (Contributed by Reiden Ong, and Pieter Eendebak in :gh:`143414` and :gh:`146306`.) The JIT optimizer now supports significantly more operations than in 3.14. (Contributed by Kumar Aditya, Ken Jin, Jiahao Li, and Sacul in :gh:`131798`.) +.. rubric:: GDB and GNU ``backtrace()`` unwinding support + +The JIT compiler now publishes unwind information for generated machine code to the +GDB interface on supported Linux ELF platforms. When libgcc frame +registration is available, the same unwind information is also registered for +GNU ``backtrace()`` stack walkers. This allows native debuggers, crash +handlers, and diagnostic tools using these mechanisms to unwind through JIT +frames instead of stopping at generated code. +(Contributed by Diego Russo and Pablo Galindo Salgado in :gh:`146071` and +:gh:`149104`.) + .. rubric:: Better machine code generation The JIT compiler's machine code generator now produces better machine code @@ -1790,6 +2084,10 @@ importlib.resources pathlib ------- +* :meth:`pathlib.Path.mkdir` now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories when ``parents=True``. + (Contributed by Gregory P. Smith in :gh:`86533`.) + * Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. Use :func:`os.path.isreserved` to detect reserved paths on Windows. (Contributed by Nikita Sobolev in :gh:`133875`.) @@ -1853,7 +2151,7 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) -* Deprecated :func:`!typing.no_type_check_decorator` has been removed. +* Deprecated :deco:`!typing.no_type_check_decorator` has been removed. (Contributed by Nikita Sobolev in :gh:`133601`.) @@ -1946,6 +2244,13 @@ New deprecations (Contributed by kishorhange111 in :gh:`148849`.) +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does *not* explicitly close the current file. + + * :mod:`re`: * :func:`re.match` and :meth:`re.Pattern.match` are now @@ -1968,10 +2273,10 @@ New deprecations * :mod:`struct`: - * Calling the ``Struct.__new__()`` without required argument now is - deprecated and will be removed in Python 3.20. Calling - :meth:`~object.__init__` method on initialized :class:`~struct.Struct` - objects is deprecated and will be removed in Python 3.20. + * Calling ``Struct.__new__()`` without a required argument is now + deprecated and will be removed in Python 3.20. Calling the + :meth:`~object.__init__` method on an initialized :class:`~struct.Struct` + object is deprecated and will be removed in Python 3.20. (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) @@ -1989,6 +2294,12 @@ New deprecations merely imported or accessed from the :mod:`!typing` module. +* :mod:`webbrowser`: + + * :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS` and scheduled for removal in Python 3.17. + (Contributed by Jeff Lyon in :gh:`137586`.) + * ``__version__`` * The ``__version__``, ``version`` and ``VERSION`` attributes have been @@ -2034,6 +2345,8 @@ New deprecations .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst .. include:: ../deprecations/soft-deprecations.rst @@ -2074,8 +2387,8 @@ New features (Contributed by Petr Viktorin in :gh:`131510`.) * Add API for checking an extension module's ABI compatibility: - :c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, :c:macro:`PyABIInfo_VAR` - and :c:data:`Py_mod_abi`. + :c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, + and :c:macro:`PyABIInfo_VAR`. (Contributed by Petr Viktorin in :gh:`137210`.) .. _whatsnew315-pybyteswriter: @@ -2098,6 +2411,11 @@ New features (Contributed by Victor Stinner in :gh:`129813`.) +* :c:type:`PyCriticalSection` and related functions are added to the Stable + ABI. + + (Contributed in :gh:`149227`.) + * Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a module from a *spec* and *initfunc*. (Contributed by Itamar Oren in :gh:`116146`.) @@ -2105,6 +2423,11 @@ New features * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add a new module export hook, + :c:func:`PyModExport_* `. + + (Contributed by Petr Viktorin in :pep:`793` and :gh:`140550`.) + * Add functions that are guaranteed to be safe for use in :c:member:`~PyTypeObject.tp_traverse` handlers: :c:func:`PyObject_GetTypeData_DuringGC`, @@ -2120,6 +2443,47 @@ New features It should only be used for debugging. (Contributed by Victor Stinner in :gh:`141070`.) +* Implement :pep:`820`: ``PySlot`` -- Unified slot system for the C API. + See :ref:`capi-slots` for documentation. + + This adds: + + * The :c:type:`PySlot` struct; + * the :c:func:`PyType_FromSlots` function; + * new slot IDs: :c:macro:`Py_slot_end`, :c:macro:`Py_slot_invalid`; + :c:macro:`Py_slot_subslots`, :c:macro:`Py_tp_slots`, + :c:macro:`Py_mod_slots`; + :c:macro:`Py_tp_name`, :c:macro:`Py_tp_basicsize`, + :c:macro:`Py_tp_extra_basicsize`, :c:macro:`Py_tp_itemsize`, + :c:macro:`Py_tp_flags`, :c:macro:`Py_tp_metaclass`, + :c:macro:`Py_tp_module`; + * convenience macros: :c:macro:`PySlot_DATA`, :c:macro:`PySlot_FUNC`, + :c:macro:`PySlot_SIZE`, :c:macro:`PySlot_INT64`, :c:macro:`PySlot_UINT64`, + :c:macro:`PySlot_STATIC_DATA`, :c:macro:`PySlot_END`, + :c:macro:`PySlot_PTR`, :c:macro:`PySlot_PTR_STATIC`. + + The :c:func:`PyModule_FromSlotsAndSpec` function and + ``PyModExport`` :ref:`module export hook ` also + use the new :c:type:`!PySlot` struct. + + These following functions are :term:`soft deprecated`: + + * :c:func:`PyType_FromSpec` + * :c:func:`PyType_FromSpecWithBases` + * :c:func:`PyType_FromModuleAndSpec` + * :c:func:`PyType_FromMetaclass` + * :c:func:`PyModule_FromDefAndSpec` + * :c:func:`PyModule_FromDefAndSpec2` + * :c:func:`PyModule_ExecDef` + + + The slots :c:macro:`Py_tp_bases` and :c:macro:`Py_tp_base` are now + equivalent: they can be set either to a single type or a tuple of types. + The :c:macro:`Py_tp_bases` slot is preferred; the other is ignored if both + are specified. + + (Contributed by Petr Viktorin in :gh:`149044`.) + * Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the stack protection base address and stack protection size of a Python @@ -2133,6 +2497,11 @@ New features Python 3.14. (Contributed by Victor Stinner in :gh:`142417`.) +* Add :c:func:`PyUnstable_DumpTraceback` and + :c:func:`PyUnstable_DumpTracebackThreads` functions to output Python + stacktraces. + (Contributed by Alex Malyshev in :gh:`145559`.) + Changed C APIs -------------- @@ -2144,14 +2513,6 @@ Changed C APIs for ``NULL`` should be updated to call :c:macro:`PyDateTime_IMPORT` instead. (Contributed by Kumar Aditya in :gh:`141563`.) -Porting to Python 3.15 ----------------------- - -* Private functions promoted to public C APIs: - - The |pythoncapi_compat_project| can be used to get most of these new - functions on Python 3.14 and older. - Removed C APIs -------------- @@ -2161,12 +2522,12 @@ Removed C APIs * :c:func:`!PyUnicode_AsDecodedObject`: Use :c:func:`PyCodec_Decode` instead. * :c:func:`!PyUnicode_AsDecodedUnicode`: - Use :c:func:`PyCodec_Decode` instead; Note that some codecs (for example, "base64") + Use :c:func:`PyCodec_Decode` instead; note that some codecs (for example, "base64") may return a type other than :class:`str`, such as :class:`bytes`. * :c:func:`!PyUnicode_AsEncodedObject`: Use :c:func:`PyCodec_Encode` instead. * :c:func:`!PyUnicode_AsEncodedUnicode`: - Use :c:func:`PyCodec_Encode` instead; Note that some codecs (for example, "base64") + Use :c:func:`PyCodec_Encode` instead; note that some codecs (for example, "base64") may return a type other than :class:`bytes`, such as :class:`str`. (Contributed by Stan Ulbrych in :gh:`133612`.) @@ -2304,7 +2665,7 @@ Build changes ============= * Removed implicit fallback to the bundled copy of the ``libmpdec`` library. - Now this should be explicitly enabled with :option:`--with-system-libmpdec` + Now this should be explicitly enabled with :option:`!--with-system-libmpdec` set to ``no`` or with :option:`!--without-system-libmpdec`. (Contributed by Sergey B Kirpichev in :gh:`115119`.) @@ -2328,14 +2689,15 @@ Build changes and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. (Contributed by Donghee Na in :gh:`141770`.) -.. _whatsnew315-frame-pointers: - * CPython is now built with frame pointers enabled by default (:pep:`831`). Pass :option:`--without-frame-pointers` to opt out. + Authors of C extensions and native libraries built with custom build - systems should add ``-fno-omit-frame-pointer`` and - ``-mno-omit-leaf-frame-pointer`` to their own ``CFLAGS`` to keep the - unwind chain intact. + systems should ensure the unwind chain is intact. + This is usually done by adding ``-fno-omit-frame-pointer`` and + similar flags to ``CFLAGS``. See :option:`--without-frame-pointers` + documentation for the specific flags Python uses. + (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in :gh:`149201`.) .. _whatsnew315-windows-tail-calling-interpreter: diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst new file mode 100644 index 000000000000000..32962a9520fa691 --- /dev/null +++ b/Doc/whatsnew/3.16.rst @@ -0,0 +1,520 @@ + +**************************** + What's new in Python 3.16 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.16, compared to 3.15. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.16 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary --- release highlights +============================== + +.. This section singles out the most important changes in Python 3.16. + Brevity is key. + + +.. PEP-sized items next. + + + +New features +============ + + + +Other language changes +====================== + + + +New modules +=========== + +* None yet. + + +Improved modules +================ + +curses +------ + +* Add support for multiple terminals to the :mod:`curses` module: + the new functions :func:`curses.newterm`, :func:`curses.set_term` + and :func:`curses.new_prescr`, + the corresponding :ref:`screen ` object, + and the :meth:`window.use() ` method. + (Contributed by Serhiy Storchaka in :gh:`90092`.) + +* The :mod:`curses` character-cell window methods now accept a full character + cell --- a spacing character optionally followed by combining characters --- + in addition to a single integer or byte character. This affects + :meth:`~curses.window.addch`, :meth:`~curses.window.bkgd`, + :meth:`~curses.window.bkgdset`, :meth:`~curses.window.border`, + :meth:`~curses.window.box`, :meth:`~curses.window.echochar`, + :meth:`~curses.window.hline`, :meth:`~curses.window.insch` and + :meth:`~curses.window.vline`. + Also add the wide-character read methods :meth:`~curses.window.get_wstr` and + :meth:`~curses.window.in_wstr`, the counterparts of + :meth:`~curses.window.getstr` and :meth:`~curses.window.instr` that return a + :class:`str` rather than :class:`bytes`, + and the module functions :func:`curses.erasewchar`, :func:`curses.killwchar` + and :func:`curses.wunctrl`, the wide-character counterparts of + :func:`curses.erasechar`, :func:`curses.killchar` and :func:`curses.unctrl`. + These features are only available when built against the wide-character + ncursesw library. + (Contributed by Serhiy Storchaka in :gh:`151757`.) + +* Add :func:`curses.nofilter`, which undoes the effect of :func:`curses.filter`. + (Contributed by Serhiy Storchaka in :gh:`151744`.) + +* Add the :mod:`curses` functions :func:`curses.alloc_pair`, + :func:`curses.find_pair`, :func:`curses.free_pair` and + :func:`curses.reset_color_pairs` for dynamic color-pair management, + available when built against a wide-character ncurses with extended-color + support. + (Contributed by Serhiy Storchaka in :gh:`151774`.) + +* Add :mod:`curses` functions and window methods that report state which could + previously only be set, such as :meth:`curses.window.is_keypad`, + :meth:`curses.window.getparent` and :func:`curses.is_cbreak`, + available when built against an ncurses with ``NCURSES_EXT_FUNCS``. + (Contributed by Serhiy Storchaka in :gh:`151776`.) + +gzip +---- + +* :func:`gzip.open` now accepts an optional argument ``mtime`` + which is passed on to the constructor of the :class:`~gzip.GzipFile` class. + (Contributed by Marin Misur in :gh:`91372`.) + + +logging +------- + +* :class:`~logging.handlers.TimedRotatingFileHandler` now uses the creation + time instead of the last modification time of an existing log file as + the basis for the first rotation after handler creation, if supported by + the OS and file system. + This allows it to be used in short-running programs that start and end + before the rotation interval expires. + (Contributed by Iván Márton and Serhiy Storchaka in :gh:`84649`.) + + +lzma +---- + +* Add support of new BCJ filters ARM64 and RISC-V via + :const:`!lzma.FILTER_ARM64` and :const:`!lzma.FILTER_RISCV`. Note that the + new filters will work only if the runtime library supports them. ARM64 filter + requires ``lzma`` 5.4.0 or newer while RISC-V requires 5.6.0 or newer. + (Contributed by Chien Wong in :gh:`115988`.) + + +math +---- + +* Added trigonometric functions that work in units of half turns, rather than + radians. The new functions :func:`math.acospi`, :func:`math.asinpi`, + :func:`math.atanpi`, and :func:`math.atan2pi` return half-turn angles. The + new functions :func:`math.cospi`, :func:`math.sinpi`, and :func:`math.tanpi` + take half-turn angle arguments. These functions are recommended by IEEE + 754-2019 and standardized in C23. + (Contributed by Jeff Epler in :gh:`150534`.) + + +os +-- + +* Add :func:`os.pidfd_getfd` for duplicating a file descriptor from another + process via a pidfd. Available on Linux 5.6+. + (Contributed by Maurycy Pawłowski-Wieroński in :gh:`149464`.) + + +re +-- + +* :mod:`re` now supports set operations and nested sets in character classes, + as described in `Unicode Technical Standard #18 + `__: set difference (``[A--B]``), + intersection (``[A&&B]``) and union (``[A||B]``), where an operand may be a + nested set written in square brackets. For example, ``[a-z--[aeiou]]`` + matches an ASCII lowercase consonant. + (Contributed by Serhiy Storchaka in :gh:`152100`.) + + +shlex +----- + +* Add keyword-only parameter *force* to :func:`shlex.quote` to force quoting + a string, even if it is already safe for a shell without being quoted. + (Contributed by Jay Berry in :gh:`148846`.) + + +tkinter +------- + +* Added many :class:`tkinter.ttk.Treeview` methods wrapping the enhanced + ``ttk::treeview`` widget commands from Tk 9.1, such as + :meth:`~tkinter.ttk.Treeview.sort`, :meth:`~tkinter.ttk.Treeview.search`, + :meth:`~tkinter.ttk.Treeview.expand`, :meth:`~tkinter.ttk.Treeview.collapse`, + :meth:`~tkinter.ttk.Treeview.hide`, :meth:`~tkinter.ttk.Treeview.unhide`, and + methods for cell focus, selection and tagging. The + :meth:`~tkinter.ttk.Treeview.expand` and :meth:`~tkinter.ttk.Treeview.collapse` + methods (without recursion) also work on Tk older than 9.1. + (Contributed by Serhiy Storchaka in :gh:`151910`.) + +* Added new :class:`!tkinter.Text` methods :meth:`~tkinter.Text.edit_canundo` + and :meth:`~tkinter.Text.edit_canredo` which return whether an undo or redo + is possible. + (Contributed by Serhiy Storchaka in :gh:`151674`.) + +* Added new :class:`!tkinter.Text` methods :meth:`~tkinter.Text.sync` and + :meth:`~tkinter.Text.pendingsync` which control and report the + synchronization of the displayed view with the underlying text. + (Contributed by Serhiy Storchaka in :gh:`151675`.) + +* Added the :meth:`ttk.Style.theme_styles + ` method which returns the list of styles + defined in a theme. + (Contributed by Serhiy Storchaka in :gh:`151920`.) + +* Added new :class:`!tkinter.Canvas` methods :meth:`~tkinter.Canvas.rchars` + which replaces the text or coordinates of canvas items, and + :meth:`~tkinter.Canvas.rotate` which rotates the coordinates of canvas items. + (Contributed by Serhiy Storchaka in :gh:`151876`.) + +* Added a :meth:`!validate` method to the :class:`!tkinter.Entry` and + :class:`!tkinter.Spinbox` widgets, which forces an evaluation of the + validation command. + (Contributed by Serhiy Storchaka in :gh:`151878`.) + +* Added the :meth:`tkinter.Menu.postcascade` method, and the + :meth:`~tkinter.Misc.tk_scaling` and :meth:`~tkinter.Misc.tk_inactive` + methods which respectively query or set the display scaling factor and + report the user idle time. + (Contributed by Serhiy Storchaka in :gh:`151881`.) + +* Added new window-management methods :meth:`~tkinter.Misc.winfo_isdark` + (dark mode detection), :meth:`~tkinter.Wm.wm_iconbadge` (application icon + badge) and :meth:`~tkinter.Wm.wm_stackorder` (toplevel stacking order). + (Contributed by Serhiy Storchaka in :gh:`151874`.) + +* Added the :meth:`~tkinter.Misc.tk_appname`, + :meth:`~tkinter.Misc.tk_useinputmethods` and :meth:`~tkinter.Misc.tk_caret` + methods, exposing the application send name, the X Input Methods state and + the input method caret location. + (Contributed by Serhiy Storchaka in :gh:`151886`.) + +* Added support for more options in :class:`!tkinter.PhotoImage` methods: the + *format* parameter of :meth:`~tkinter.PhotoImage.put`, the *metadata* + parameter of :meth:`~tkinter.PhotoImage.put`, :meth:`~tkinter.PhotoImage.read`, + :meth:`~tkinter.PhotoImage.write` and :meth:`~tkinter.PhotoImage.data`, and + the *withalpha* parameter of :meth:`~tkinter.PhotoImage.get`. + (Contributed by Serhiy Storchaka in :gh:`151890`.) + +* Added the :meth:`~tkinter.PhotoImage.redither` method which recalculates the + dithered image when its data was supplied in pieces. + (Contributed by Serhiy Storchaka in :gh:`151888`.) + +* :class:`tkinter.OptionMenu` now accepts arbitrary :class:`!tkinter.Menubutton` + options as keyword arguments, which can also override its default appearance. + (Contributed by Serhiy Storchaka in :gh:`101284`.) + +* The :mod:`tkinter.simpledialog` dialogs were modernized to match the look + and feel of the native Tk dialogs. + :class:`!tkinter.simpledialog.SimpleDialog` and the + :func:`~tkinter.simpledialog.askinteger`, + :func:`~tkinter.simpledialog.askfloat` and + :func:`~tkinter.simpledialog.askstring` dialogs are now built from the themed + :mod:`tkinter.ttk` widgets instead of the classic :mod:`tkinter` widgets; + the :class:`!tkinter.simpledialog.Dialog` base class still defaults to the + classic widgets for compatibility. Both :class:`!Dialog` and + :class:`!SimpleDialog` gained a *use_ttk* parameter that selects between the + classic Tk widgets and the themed ttk widgets. :class:`!SimpleDialog` also + gained *bitmap* and + *detail* parameters, draws the standard icons with themed images in the + ttk version, and accepts mappings of button options as *buttons* entries. + (Contributed by Serhiy Storchaka in :gh:`59396`.) + +xml +--- + +* Add support for multiple multi-byte encodings in the :mod:`XML parser + `: "cp932", "cp949", "cp950", "Big5", "EUC-JP", + "GB2312", "GBK", "johab", and "Shift_JIS". + Add partial support (only BMP characters) for multi-byte encodings + "Big5-HKSCS", "EUC_JIS-2004", "EUC_JISX0213", "Shift_JIS-2004", + "Shift_JISX0213", "utf-8-sig" and non-standard aliases like "UTF8" + (without hyphen). + The parser now raises :exc:`ValueError` for known unsupported + multi-byte encodings such as "ISO-2022-JP" or "raw-unicode-escape" + instead of failing later, when encountering non-ASCII data. + (Contributed by Serhiy Storchaka in :gh:`62259`.) + +zipfile +------- + +* Add :meth:`ZipFile.remove() ` to remove a member + from an archive's central directory, and + :meth:`ZipFile.repack() ` to reclaim the space used + by the local file entries of removed members. + (Contributed by Danny Lin in :gh:`51067`.) + +.. Add improved modules above alphabetically, not here at the end. + +Optimizations +============= + +re +-- + +* Character class escapes (``\d``, ``\D``, ``\s``, ``\S``, ``\w`` and ``\W``) + outside a character set, and character sets containing a single such escape + (such as ``[\d]`` or ``[^\s]``), are now compiled to a single ``CATEGORY`` + opcode instead of being wrapped in an ``IN`` block. This speeds up matching + of patterns such as ``\d+`` and reduces the size of the compiled byte code. + (Contributed by Serhiy Storchaka in :gh:`152033` and Pieter Eendebak in + :gh:`152056`.) + +module_name +----------- + +* TODO + + + +Removed +======= + +annotationlib +------------- + +* The :meth:`!annotationlib.ForwardRef._evaluate` method + which has been deprecated since Python 3.14. + Use :meth:`annotationlib.ForwardRef.evaluate` + or :func:`typing.evaluate_forward_ref` instead. + +array +----- + +* The ``'u'`` format code (:c:type:`wchar_t`) which has been deprecated in + documentation since Python 3.3 and at runtime since Python 3.13. + Use ``'w'`` format code instead (:c:type:`Py_UCS4`, always 4 bytes). + +asyncio +------- + +* The :func:`!asyncio.iscoroutinefunction` + which has been deprecated since Python 3.14. + Use :func:`inspect.iscoroutinefunction` instead. + +functools +--------- + +* Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments has been deprecated since Python 3.14. + +logging +------- + +* Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. + +mimetypes +--------- + +* Valid extensions start with a '.' or are empty for + :meth:`mimetypes.MimeTypes.add_type`. + Undotted extensions now raise a :exc:`ValueError`. + +shutil +------ + +* The :exc:`!ExecError` exception which has been deprecated since Python 3.14. + It has not been used by any function in :mod:`!shutil` since Python 3.4. + (Contributed by Stan Ulbrych in :gh:`149567`.) + +symtable +-------- + +* The :meth:`!symtable.Class.get_methods` method + which has been deprecated since Python 3.14. + +sys +--- + +* The :func:`!_enablelegacywindowsfsencoding` function + which has been deprecated since Python 3.13. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + (Contributed by Stan Ulbrych in :gh:`149595`.) + +sysconfig +--------- + +* The :func:`!sysconfig.expand_makefile_vars` function + which has been deprecated since Python 3.14. + Use the ``vars`` argument of :func:`sysconfig.get_paths` instead. + (Contributed by Stan Ulbrych in :gh:`149499`.) + +tarfile +------- + +* The undocumented and unused :attr:`!tarfile.TarFile.tarfile` attribute + has been deprecated since Python 3.13. + +.. Add removals above alphabetically, not here at the end. + + +Deprecated +========== + +New deprecations +---------------- + +* :mod:`abc` + + * Soft-deprecated since Python 3.3 :class:`abc.abstractclassmethod`, + :class:`abc.abstractstaticmethod`, and :class:`abc.abstractproperty` + now raise a :exc:`DeprecationWarning`. + These classes will be removed in Python 3.21, instead + use :func:`abc.abstractmethod` with :func:`classmethod`, + :func:`staticmethod`, and :class:`property` respectively. + +* :mod:`ast`: + + * Classes ``slice``, ``Index``, ``ExtSlice``, ``Suite``, ``Param``, + ``AugLoad`` and ``AugStore``, deprecated since Python 3.9, are no longer + imported by ``from ast import *`` and issue a deprecation warning on + use. The classes are slated for removal in Python 3.21. These types are not + generated by the parser or accepted by the code generator. + * The ``dims`` property of ``ast.Tuple`` objects, deprecated since Python + 3.9, now issues a deprecation warning on use. This property is slated for + removal in 3.21. Use ``ast.Tuple.elts`` instead. + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.17.rst + +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-3.21.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + + +Porting to Python 3.16 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Build changes +============= + +* Remove the bundled copy of the libmpdec_ decimal library from the CPython source tree + to simplify maintenance and updates. The :mod:`decimal` module will now + unconditionally use the system's libmpdec decimal library. Also remove the + now unused :option:`!--with-system-libmpdec` :program:`configure` flag. + This change has no impact on binary releases of Python, which have been + built against a separate copy of libmpdec for the past several releases. + + (Contributed by Sergey B Kirpichev in :gh:`115119`.) + + .. _libmpdec: https://www.bytereef.org/mpdecimal/ + +* Add a :option:`--with-build-details-suffix` configure flag to allow + Linux distributions that co-install multiple versions of Python in the + same tree to avoid ``build-details.json`` clashes. + + (Contributed by Stefano Rivera in :gh:`131372`.) + + +C API changes +============= + +New features +------------ + +* TODO + +Porting to Python 3.16 +---------------------- + +* TODO + +Deprecated C APIs +----------------- + +* :c:func:`PyGen_New`, :c:func:`PyGen_NewWithQualName`, :c:func:`PyCoro_New`, + and :c:func:`PyAsyncGen_New` are deprecated. + They are scheduled for removal in 3.18. + +.. Add C API deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.19.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.20.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + +Removed C APIs +-------------- diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 48c461d891e8614..c61eb659ccc0f62 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -756,7 +756,7 @@ functools --------- * The :mod:`functools` module includes a new decorator for caching function - calls. :func:`functools.lru_cache` can save repeated queries to an external + calls. :deco:`functools.lru_cache` can save repeated queries to an external resource whenever the results are expected to be the same. For example, adding a caching decorator to a database query function can save @@ -789,7 +789,7 @@ functools `_\, :issue:`10586`, and :issue:`10593`.) -* The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute +* The :deco:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute pointing to the original callable function. This allows wrapped functions to be introspected. It also copies :attr:`~function.__annotations__` if defined. And now it also gracefully skips over missing attributes such as @@ -805,11 +805,11 @@ functools :issue:`8814`.) * To help write classes with rich comparison methods, a new decorator - :func:`functools.total_ordering` will use existing equality and inequality + :deco:`functools.total_ordering` will use existing equality and inequality methods to fill in the remaining methods. For example, supplying *__eq__* and *__lt__* will enable - :func:`~functools.total_ordering` to fill-in *__le__*, *__gt__* and *__ge__*:: + :deco:`~functools.total_ordering` to fill-in *__le__*, *__gt__* and *__ge__*:: @total_ordering class Student: @@ -1097,11 +1097,11 @@ logarithm of the gamma function: abc --- -The :mod:`abc` module now supports :func:`~abc.abstractclassmethod` and -:func:`~abc.abstractstaticmethod`. +The :mod:`abc` module now supports :deco:`~abc.abstractclassmethod` and +:deco:`~abc.abstractstaticmethod`. These tools make it possible to define an :term:`abstract base class` that -requires a particular :func:`classmethod` or :func:`staticmethod` to be +requires a particular :deco:`classmethod` or :deco:`staticmethod` to be implemented:: class Temperature(metaclass=abc.ABCMeta): @@ -1155,7 +1155,7 @@ self-reference by displaying "..." in the recursive part of the representation string. To help write such :meth:`~object.__repr__` methods, the :mod:`reprlib` module has a new -decorator, :func:`~reprlib.recursive_repr`, for detecting recursive calls to +decorator, :deco:`~reprlib.recursive_repr`, for detecting recursive calls to :meth:`!__repr__` and substituting a placeholder string instead:: >>> class MyList(list): @@ -1241,7 +1241,7 @@ There is a new and slightly mind-blowing tool :term:`context manager` that does double duty as a function decorator. As a convenience, this new functionality is used by -:func:`~contextlib.contextmanager` so that no extra effort is needed to support +:deco:`~contextlib.contextmanager` so that no extra effort is needed to support both roles. The basic idea is that both context managers and function decorators can be used @@ -1253,7 +1253,7 @@ write a pre-action or post-action wrapper that can be used in either role. For example, it is sometimes useful to wrap functions or groups of statements with a logger that can track the time of entry and time of exit. Rather than writing both a function decorator and a context manager for the task, the -:func:`~contextlib.contextmanager` provides both capabilities in a single +:deco:`~contextlib.contextmanager` provides both capabilities in a single definition:: from contextlib import contextmanager diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 1bb79bce2c3e972..02fd264e53e1b06 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -919,12 +919,12 @@ abstract methods. The recommended approach to declaring abstract descriptors is now to provide :attr:`!__isabstractmethod__` as a dynamically updated property. The built-in descriptors have been updated accordingly. -* :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. -* :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. -* :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. +* :deco:`abc.abstractproperty` has been deprecated, use :deco:`property` + with :deco:`abc.abstractmethod` instead. +* :deco:`abc.abstractclassmethod` has been deprecated, use + :deco:`classmethod` with :deco:`abc.abstractmethod` instead. +* :deco:`abc.abstractstaticmethod` has been deprecated, use + :deco:`staticmethod` with :deco:`abc.abstractmethod` instead. (Contributed by Darren Dale in :issue:`11610`.) @@ -1390,7 +1390,7 @@ ftplib functools --------- -The :func:`functools.lru_cache` decorator now accepts a ``typed`` keyword +The :deco:`functools.lru_cache` decorator now accepts a ``typed`` keyword argument (that defaults to ``False`` to ensure that it caches values of different types that compare equal in separate cache slots. (Contributed by Raymond Hettinger in :issue:`13227`.) @@ -2244,12 +2244,12 @@ Deprecated Python modules, functions and methods * The :func:`!os.stat_float_times` function is deprecated. * :mod:`abc` module: - * :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. + * :deco:`abc.abstractproperty` has been deprecated, use :deco:`property` + with :deco:`abc.abstractmethod` instead. + * :deco:`abc.abstractclassmethod` has been deprecated, use + :deco:`classmethod` with :deco:`abc.abstractmethod` instead. + * :deco:`abc.abstractstaticmethod` has been deprecated, use + :deco:`staticmethod` with :deco:`abc.abstractmethod` instead. * :mod:`importlib` package: diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index a390211ddb50215..0067491c569cc05 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -858,7 +858,7 @@ to behave like normal instance methods when included in a class definition. .. _whatsnew-singledispatch: -The new :func:`~functools.singledispatch` decorator brings support for +The new :deco:`~functools.singledispatch` decorator brings support for single-dispatch generic functions to the Python standard library. Where object oriented programming focuses on grouping multiple operations on a common set of data into a class, a generic function focuses on grouping @@ -870,7 +870,7 @@ multiple implementations of an operation that allows it to work with :pep:`443` -- Single-dispatch generic functions PEP written and implemented by Łukasz Langa. -:func:`~functools.total_ordering` now supports a return value of +:deco:`~functools.total_ordering` now supports a return value of :data:`NotImplemented` from the underlying comparison function. (Contributed by Katie Miller in :issue:`10042`.) @@ -1029,7 +1029,7 @@ information for modules, classes and functions. (Contributed by Claudiu Popa and Nick Coghlan in :issue:`18626`.) :func:`~inspect.unwrap` makes it easy to unravel wrapper function chains -created by :func:`functools.wraps` (and any other API that sets the +created by :deco:`functools.wraps` (and any other API that sets the ``__wrapped__`` attribute on a wrapper function). (Contributed by Daniel Urban, Aaron Iles and Nick Coghlan in :issue:`13266`.) @@ -2317,7 +2317,7 @@ Changes in the Python API wish to continue to ignore syntax or decoding issues, catch all three exceptions now. -* :func:`functools.update_wrapper` and :func:`functools.wraps` now correctly +* :func:`functools.update_wrapper` and :deco:`functools.wraps` now correctly set the ``__wrapped__`` attribute to the function being wrapped, even if that function also had its ``__wrapped__`` attribute set. This means ``__wrapped__`` attributes now correctly link a stack of decorated diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 6009dd8a71eea52..24a2ffbefd0cfba 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -121,7 +121,7 @@ Significant improvements in the standard library: :ref:`better and significantly faster way ` of directory traversal. -* :func:`functools.lru_cache` has been mostly +* :deco:`functools.lru_cache` has been mostly :ref:`reimplemented in C `, yielding much better performance. @@ -1148,7 +1148,7 @@ functools .. _whatsnew-lrucache: -Most of the :func:`~functools.lru_cache` machinery is now implemented in C, making +Most of the :deco:`~functools.lru_cache` machinery is now implemented in C, making it significantly faster. (Contributed by Matt Joiner, Alexey Kachayev, and Serhiy Storchaka in :issue:`14373`.) @@ -2158,7 +2158,7 @@ improvement in some benchmarks. Objects from the :mod:`random` module now use 50% less memory on 64-bit builds. (Contributed by Serhiy Storchaka in :issue:`23488`.) -The :func:`property` getter calls are up to 25% faster. +The :deco:`property` getter calls are up to 25% faster. (Contributed by Joe Jevnik in :issue:`23910`.) Instantiation of :class:`fractions.Fraction` is now up to 30% faster. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index bdd35d39e361949..b0382dd8215bc00 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -517,7 +517,7 @@ Applications that do not use str to represent paths should use :func:`os.fsencode` and :func:`os.fsdecode` to ensure their bytes are correctly encoded. To revert to the previous behaviour, set :envvar:`PYTHONLEGACYWINDOWSFSENCODING` or call -:func:`sys._enablelegacywindowsfsencoding`. +:func:`!sys._enablelegacywindowsfsencoding`. See :pep:`529` for more information and discussion of code modifications that may be required. diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 5dd47cdac96a5c0..3af3e6ec9cac08f 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -571,7 +571,7 @@ decimal operations to work with the correct context in asynchronous code. dataclasses ----------- -The new :func:`~dataclasses.dataclass` decorator provides a way to declare +The new :deco:`~dataclasses.dataclass` decorator provides a way to declare *data classes*. A data class describes its attributes using class variable annotations. Its constructor and other magic methods, such as :meth:`~object.__repr__`, :meth:`~object.__eq__`, and @@ -830,7 +830,7 @@ The new :func:`~contextlib.nullcontext` is a simpler and faster no-op context manager than :class:`~contextlib.ExitStack`. (Contributed by Jesse-Bakker in :issue:`10049`.) -The new :func:`~contextlib.asynccontextmanager`, +The new :deco:`~contextlib.asynccontextmanager`, :class:`~contextlib.AbstractAsyncContextManager`, and :class:`~contextlib.AsyncExitStack` have been added to complement their synchronous counterparts. (Contributed @@ -921,7 +921,7 @@ return :const:`False` instead and are deprecated. functools --------- -:func:`functools.singledispatch` now supports registering implementations +:deco:`functools.singledispatch` now supports registering implementations using type annotations. (Contributed by Łukasz Langa in :issue:`32227`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 5078fc30ac111e4..42d98ff94ec0d1b 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -764,7 +764,7 @@ these are the inverse of each class's ``isocalendar`` method. functools --------- -:func:`functools.lru_cache` can now be used as a straight decorator rather +:deco:`functools.lru_cache` can now be used as a straight decorator rather than as a function returning a decorator. So both of these are now supported:: @lru_cache @@ -777,7 +777,7 @@ than as a function returning a decorator. So both of these are now supported:: (Contributed by Raymond Hettinger in :issue:`36772`.) -Added a new :func:`functools.cached_property` decorator, for computed properties +Added a new :deco:`functools.cached_property` decorator, for computed properties cached for the life of the instance. :: import functools @@ -893,7 +893,7 @@ inspect The :func:`inspect.getdoc` function can now find docstrings for ``__slots__`` if that attribute is a :class:`dict` where the values are docstrings. This provides documentation options similar to what we already have -for :func:`property`, :func:`classmethod`, and :func:`staticmethod`:: +for :deco:`property`, :deco:`classmethod`, and :deco:`staticmethod`:: class AudioClip: __slots__ = {'bit_rate': 'expressed in kilohertz to one decimal place', @@ -1333,14 +1333,14 @@ The :mod:`typing` module incorporates several new features: ... * "Final" variables, functions, methods and classes. See :pep:`591`, - :class:`typing.Final` and :func:`typing.final`. + :class:`typing.Final` and :deco:`typing.final`. The final qualifier instructs a static type checker to restrict subclassing, overriding, or reassignment:: pi: Final[float] = 3.1415926536 * Protocol definitions. See :pep:`544`, :class:`typing.Protocol` and - :func:`typing.runtime_checkable`. Simple ABCs like + :deco:`typing.runtime_checkable`. Simple ABCs like :class:`typing.SupportsInt` are now ``Protocol`` subclasses. * New protocol class :class:`typing.SupportsIndex`. diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index 38194db670b8390..420876a9b3ecd00 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.16.rst 3.15.rst 3.14.rst 3.13.rst diff --git a/Include/Python.h b/Include/Python.h index d5e38b8b0201eec..337119c15fe8b6f 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -79,7 +79,8 @@ __pragma(warning(disable: 4201)) #include "object.h" #include "refcount.h" #include "objimpl.h" -#include "typeslots.h" +#include "slots.h" +#include "slots_generated.h" #include "pyhash.h" #include "cpython/pydebug.h" #include "bytearrayobject.h" diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 4fc46fefb93a24a..bcba32da412f326 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -2,15 +2,6 @@ # error "this header file must not be included directly" #endif -// Python critical sections -// -// Conceptually, critical sections are a deadlock avoidance layer on top of -// per-object locks. These helpers, in combination with those locks, replace -// our usage of the global interpreter lock to provide thread-safety for -// otherwise thread-unsafe objects, such as dict. -// -// NOTE: These APIs are no-ops in non-free-threaded builds. -// // Straightforward per-object locking could introduce deadlocks that were not // present when running with the GIL. Threads may hold locks for multiple // objects simultaneously because Python operations can nest. If threads were @@ -43,52 +34,19 @@ // `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) // critical section by reacquiring the associated lock or locks. See // `_PyCriticalSection_Resume()`. -// -// NOTE: Only the top-most critical section is guaranteed to be active. -// Operations that need to lock two objects at once must use -// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections -// to lock more than one object at once, because the inner critical section -// may suspend the outer critical sections. This API does not provide a way -// to lock more than two objects at once (though it could be added later -// if actually needed). -// -// NOTE: Critical sections implicitly behave like reentrant locks because -// attempting to acquire the same lock will suspend any outer (earlier) -// critical sections. However, they are less efficient for this use case than -// purposefully designed reentrant locks. -// -// Example usage: -// Py_BEGIN_CRITICAL_SECTION(op); -// ... -// Py_END_CRITICAL_SECTION(); -// -// To lock two objects at once: -// Py_BEGIN_CRITICAL_SECTION2(op1, op2); -// ... -// Py_END_CRITICAL_SECTION2(); - -typedef struct PyCriticalSection PyCriticalSection; -typedef struct PyCriticalSection2 PyCriticalSection2; - -PyAPI_FUNC(void) -PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); PyAPI_FUNC(void) PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); -PyAPI_FUNC(void) -PyCriticalSection_End(PyCriticalSection *c); - -PyAPI_FUNC(void) -PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); - PyAPI_FUNC(void) PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); -PyAPI_FUNC(void) -PyCriticalSection2_End(PyCriticalSection2 *c); - #ifndef Py_GIL_DISABLED +#undef Py_BEGIN_CRITICAL_SECTION +#undef Py_END_CRITICAL_SECTION +#undef Py_BEGIN_CRITICAL_SECTION2 +#undef Py_END_CRITICAL_SECTION2 + # define Py_BEGIN_CRITICAL_SECTION(op) \ { # define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ @@ -101,54 +59,17 @@ PyCriticalSection2_End(PyCriticalSection2 *c); { # define Py_END_CRITICAL_SECTION2() \ } -#else /* !Py_GIL_DISABLED */ - -// NOTE: the contents of this struct are private and may change betweeen -// Python releases without a deprecation period. -struct PyCriticalSection { - // Tagged pointer to an outer active critical section (or 0). - uintptr_t _cs_prev; - - // Mutex used to protect critical section - PyMutex *_cs_mutex; -}; - -// A critical section protected by two mutexes. Use -// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. -// NOTE: the contents of this struct are private and may change betweeen -// Python releases without a deprecation period. -struct PyCriticalSection2 { - PyCriticalSection _cs_base; - PyMutex *_cs_mutex2; -}; - -# define Py_BEGIN_CRITICAL_SECTION(op) \ - { \ - PyCriticalSection _py_cs; \ - PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +#else /* !Py_GIL_DISABLED */ # define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ { \ PyCriticalSection _py_cs; \ PyCriticalSection_BeginMutex(&_py_cs, mutex) -# define Py_END_CRITICAL_SECTION() \ - PyCriticalSection_End(&_py_cs); \ - } - -# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ - { \ - PyCriticalSection2 _py_cs2; \ - PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) - # define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ { \ PyCriticalSection2 _py_cs2; \ PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) -# define Py_END_CRITICAL_SECTION2() \ - PyCriticalSection2_End(&_py_cs2); \ - } - -#endif +#endif /* !Py_GIL_DISABLED */ diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index f75884e597e2c24..e14facd77edec53 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -16,8 +16,8 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; #define PyGen_Check(op) PyObject_TypeCheck((op), &PyGen_Type) #define PyGen_CheckExact(op) Py_IS_TYPE((op), &PyGen_Type) -PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); -PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, PyObject *name, PyObject *qualname); PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); @@ -29,7 +29,7 @@ typedef struct _PyCoroObject PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; #define PyCoro_CheckExact(op) Py_IS_TYPE((op), &PyCoro_Type) -PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); @@ -40,7 +40,7 @@ typedef struct _PyAsyncGenObject PyAsyncGenObject; PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; -PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *name, PyObject *qualname); #define PyAsyncGen_CheckExact(op) Py_IS_TYPE((op), &PyAsyncGen_Type) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index d0b3d0cb14de3df..326254c335b4895 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -241,7 +241,11 @@ struct _typeobject { * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). */ uint16_t tp_versions_used; - _Py_iteritemfunc _tp_iteritem; /* Virtual iterator next function */ + + /* Virtual iterator next function. + * This function must escape to any code that can result in + * the GC being run, such as Py_DECREF. */ + _Py_iteritemfunc _tp_iteritem; }; #define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0cb57679df331d9..a9d97e47e005dff 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -105,7 +105,7 @@ struct _ts { # define _PyThreadState_WHENCE_INIT 1 # define _PyThreadState_WHENCE_FINI 2 # define _PyThreadState_WHENCE_THREADING 3 -# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_C_API 4 # define _PyThreadState_WHENCE_EXEC 5 # define _PyThreadState_WHENCE_THREADING_DAEMON 6 #endif @@ -239,6 +239,20 @@ struct _ts { // structure and all share the same per-interpreter structure). PyStats *pystats; #endif + + struct { + /* Number of nested PyThreadState_Ensure() calls on this thread state */ + Py_ssize_t counter; + + /* Should this thread state be deleted upon calling + PyThreadState_Release() (with the counter at 1)? + + This is only true for thread states created by PyThreadState_Ensure() */ + int delete_on_release; + + /* The interpreter guard owned by PyThreadState_EnsureFromView(), if any. */ + PyInterpreterGuard *owned_guard; + } ensure; }; /* other API */ diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 5d1f44988a6df10..69659c48a3bf882 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -163,6 +163,7 @@ typedef struct _optimization_stats { uint64_t jit_code_size; uint64_t jit_trampoline_size; uint64_t jit_data_size; + uint64_t jit_got_size; uint64_t jit_padding_size; uint64_t jit_freed_memory_size; uint64_t trace_total_memory_hist[_Py_UOP_HIST_SIZE]; diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h index 0b6ff0f17e6f8c1..e621d6abbfed8aa 100644 --- a/Include/cpython/sentinelobject.h +++ b/Include/cpython/sentinelobject.h @@ -1,22 +1,26 @@ /* Sentinel object interface */ #ifndef Py_LIMITED_API -#ifndef Py_SENTINELOBJECT_H -#define Py_SENTINELOBJECT_H +#ifndef _Py_SENTINELOBJECT_H +#define _Py_SENTINELOBJECT_H #ifdef __cplusplus extern "C" { #endif PyAPI_DATA(PyTypeObject) PySentinel_Type; -#define PySentinel_Check(op) Py_IS_TYPE((op), &PySentinel_Type) +#define PySentinel_CheckExact(op) Py_IS_TYPE((op), &PySentinel_Type) + +/* Alias as long as subclasses are not allowed. */ +#define PySentinel_Check(op) PySentinel_CheckExact(op) PyAPI_FUNC(PyObject *) PySentinel_New( const char *name, - const char *module_name); + const char *module_name, + const char *repr); #ifdef __cplusplus } #endif -#endif /* !Py_SENTINELOBJECT_H */ +#endif /* !_Py_SENTINELOBJECT_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h index 4c3ea1facebc4ed..137206eff15b33f 100644 --- a/Include/cpython/sliceobject.h +++ b/Include/cpython/sliceobject.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_SLICEOBJECT_H +#ifndef _Py_CPYTHON_SLICEOBJECT_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h index 328fbe86143b024..83a1abcd6f3b347 100644 --- a/Include/cpython/structseq.h +++ b/Include/cpython/structseq.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_STRUCTSEQ_H +#ifndef _Py_CPYTHON_STRUCTSEQ_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index 81c51944f136f29..7f42730f1b09194 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -11,3 +11,11 @@ struct _traceback { int tb_lasti; int tb_lineno; }; + +PyAPI_FUNC(const char*) PyUnstable_DumpTraceback(int fd, PyThreadState *tstate); + +PyAPI_FUNC(const char*) PyUnstable_DumpTracebackThreads( + int fd, + PyInterpreterState *interp, + PyThreadState *current_tstate, + Py_ssize_t max_threads); diff --git a/Include/critical_section.h b/Include/critical_section.h index 3b37615a8b17e2a..732bfab7ecf2349 100644 --- a/Include/critical_section.h +++ b/Include/critical_section.h @@ -4,6 +4,91 @@ extern "C" { #endif +// Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + +// NOTE: the contents of this struct are private and their meaning may +// change betweeen Python releases without a deprecation period. +typedef struct PyCriticalSection { + // Tagged pointer to an outer active critical section (or 0). + uintptr_t _cs_prev; + + // Mutex used to protect critical section + struct PyMutex *_cs_mutex; +} PyCriticalSection; + +// A critical section protected by two mutexes. Use +// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +typedef struct PyCriticalSection2 { + PyCriticalSection _cs_base; + + struct PyMutex *_cs_mutex2; +} PyCriticalSection2; + +PyAPI_FUNC(void) +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); + +PyAPI_FUNC(void) +PyCriticalSection_End(PyCriticalSection *c); + +PyAPI_FUNC(void) +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); + +PyAPI_FUNC(void) +PyCriticalSection2_End(PyCriticalSection2 *c); + +// These are definitions for the stable ABI. For GIL-ful builds they're +// conditionally redefined as no-ops in cpython/critical_section.h. + +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) + +# define Py_END_CRITICAL_SECTION() \ + PyCriticalSection_End(&_py_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +# define Py_END_CRITICAL_SECTION2() \ + PyCriticalSection2_End(&_py_cs2); \ + } + #ifndef Py_LIMITED_API # define Py_CPYTHON_CRITICAL_SECTION_H # include "cpython/critical_section.h" diff --git a/Include/dynamic_annotations.h b/Include/dynamic_annotations.h index 4d4def9bf8983e2..5290319dd762c55 100644 --- a/Include/dynamic_annotations.h +++ b/Include/dynamic_annotations.h @@ -461,6 +461,7 @@ int RunningOnValgrind(void); #if DYNAMIC_ANNOTATIONS_ENABLED != 0 && defined(__cplusplus) +extern "C++" { /* _Py_ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. Instead of doing @@ -476,6 +477,8 @@ int RunningOnValgrind(void); _Py_ANNOTATE_IGNORE_READS_END(); return res; } +} + /* Apply _Py_ANNOTATE_BENIGN_RACE_SIZED to a static variable. */ #define _Py_ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ namespace { \ diff --git a/Include/exports.h b/Include/exports.h index a863ecb33078aba..18692283005e596 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -103,7 +103,7 @@ #define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* #endif #ifndef PyMODEXPORT_FUNC - #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* + #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PySlot* #endif #endif /* Py_EXPORTS_H */ diff --git a/Include/import.h b/Include/import.h index 6f1c13787b8569a..c062e46bff46bf7 100644 --- a/Include/import.h +++ b/Include/import.h @@ -90,8 +90,7 @@ PyAPI_FUNC(int) PyImport_AppendInittab( typedef enum { PyImport_LAZY_NORMAL, - PyImport_LAZY_ALL, - PyImport_LAZY_NONE + PyImport_LAZY_ALL } PyImport_LazyImportsMode; #ifndef Py_LIMITED_API diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index b9eb4fd9891e669..67c6fa7c0c4ed5b 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -60,6 +60,14 @@ PyAPI_FUNC(int) _Py_convert_optional_to_non_negative_ssize_t(PyObject *, void *) // Export for 'math' shared extension. PyAPI_FUNC(PyObject*) _PyNumber_Index(PyObject *o); +typedef struct { + PyObject *object; + PySendResult kind; +} PySendResultPair; + +// Same as PyIter_Send but returns a struct for MSVC tailcall support +PyAPI_FUNC(PySendResultPair) _PyIter_Send(PyObject *iter, PyObject *arg); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 177e6d10134adbe..27a7a46152f57b8 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -62,9 +62,11 @@ _PyBytes_ReverseFind(const char *haystack, Py_ssize_t len_haystack, // // Export for 'array' shared extension. PyAPI_FUNC(void) -_PyBytes_Repeat(char* dest, Py_ssize_t len_dest, +_PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); +PyAPI_FUNC(PyObject *) _PyBytes_Repeat(PyObject *self, Py_ssize_t n); + /* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation for a bytes object of length n should request PyBytesObject_SIZE + n bytes. diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index fd4221f0816d24b..06c4ca1619d7ce1 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -208,16 +208,16 @@ extern void _PyEval_DeactivateOpCache(void); /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { +static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - // Overflow if stack pointer is between soft limit and the base of the hardware stack. - // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. - // We could have the wrong stack limits because of limited platform support, or user-space threads. + // Possible overflow if stack pointer is beyond the soft limit. + // _Py_CheckRecursiveCall will check for corner cases and + // report an error if there is an overflow. #if _Py_STACK_GROWS_DOWN - return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; + return here_addr < _tstate->c_stack_soft_limit; #else - return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; + return here_addr > _tstate->c_stack_soft_limit; #endif } @@ -232,7 +232,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCallPy( static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, const char *where) { - return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); + return (_Py_ReachedRecursionLimit(tstate) && _Py_CheckRecursiveCall(tstate, where)); } static inline int _Py_EnterRecursiveCall(const char *where) { @@ -246,8 +246,6 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate); -PyAPI_FUNC(int) _Py_ReachedRecursionLimit(PyThreadState *tstate); - // Export for test_peg_generator PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( PyThreadState *tstate, diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 4584f1bde7974e8..5b1fddbe15b98b8 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -539,7 +539,8 @@ typedef struct { PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); -extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(_Py_CODEUNIT) _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index 52dca1362592d69..bfa10eadf735731 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -45,7 +45,7 @@ extern int _PyCodec_UnregisterError(const char *name); in Python 3.5+? */ -extern PyObject* _PyCodec_LookupTextEncoding( +PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding( const char *encoding, const char *alternate_command); diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 2a2846b1296b901..51d99d74ca159f6 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -196,10 +196,9 @@ _PyCriticalSection2_Begin(PyThreadState *tstate, PyCriticalSection2 *c, PyObject static inline void _PyCriticalSection2_End(PyThreadState *tstate, PyCriticalSection2 *c) { - // if mutex1 is NULL, we used the fast path in - // _PyCriticalSection_BeginSlow for mutexes that are already held, - // which should only happen when mutex1 and mutex2 were the same mutex, - // and mutex2 should also be NULL. + // if mutex1 is NULL, we used the fast path in either + // _PyCriticalSection_BeginSlow or _PyCriticalSection2_BeginSlow for mutexes + // that are already held, and mutex2 should also be NULL. if (c->_cs_base._cs_mutex == NULL) { assert(c->_cs_mutex2 == NULL); return; diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 1dd10f8d94cfd85..18490f98a918a73 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -158,8 +158,16 @@ typedef struct _Py_DebugOffsets { uint64_t tp_name; uint64_t tp_repr; uint64_t tp_flags; + uint64_t tp_basicsize; + uint64_t tp_dictoffset; } type_object; + // PyHeapTypeObject offset; + struct _heap_type_object { + uint64_t size; + uint64_t ht_cached_keys; + } heap_type_object; + // PyTuple object offset; struct _tuple_object { uint64_t size; @@ -330,6 +338,12 @@ typedef struct _Py_DebugOffsets { .tp_name = offsetof(PyTypeObject, tp_name), \ .tp_repr = offsetof(PyTypeObject, tp_repr), \ .tp_flags = offsetof(PyTypeObject, tp_flags), \ + .tp_basicsize = offsetof(PyTypeObject, tp_basicsize), \ + .tp_dictoffset = offsetof(PyTypeObject, tp_dictoffset), \ + }, \ + .heap_type_object = { \ + .size = sizeof(PyHeapTypeObject), \ + .ht_cached_keys = offsetof(PyHeapTypeObject, ht_cached_keys), \ }, \ .tuple_object = { \ .size = sizeof(PyTupleObject), \ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index ff6588b3e9718c6..9e11c0bb546b685 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -90,6 +90,8 @@ typedef struct { } PyDictUnicodeEntry; extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *); +extern void _PyDict_RemoveKeysForClass(PyHeapTypeObject *); +extern void _PyDict_SplitKeysInvalidated(PyDictKeysObject* keys); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); /* Implementations of the `|` and `|=` operators for dict, used by the @@ -239,6 +241,17 @@ struct _dictkeysobject { see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */ }; +struct _instancekeysobject { + PyTypeObject* dsk_owning_type; + struct _dictkeysobject dsk_keys; +}; + +static inline struct _instancekeysobject *_PyDictKeys_AsSharedKeys(struct _dictkeysobject *keys) +{ + assert(keys->dk_kind == DICT_KEYS_SPLIT); + return _Py_CONTAINER_OF(keys, struct _instancekeysobject, dsk_keys); +} + /* This must be no more than 250, for the prefix size to fit in one byte. */ #define SHARED_KEYS_MAX_SIZE 30 #define NEXT_LOG2_SHARED_KEYS_MAX_SIZE 6 diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 11932b8d1e1ab60..bb6fe2625975596 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -13,6 +13,8 @@ extern "C" { struct _Py_dict_state { uint32_t next_keys_version; + PyMutex watcher_mutex; // Protects the watchers array (free-threaded builds) + _PyOnceFlag watcher_setup_once; // One-time optimizer watcher setup PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index e105677cd2e674a..84cbb56a9192156 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -223,12 +223,14 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyGC_Head *generation0 = _PyInterpreterState_GET()->gc.generation0; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + PyGC_Head *generation0 = gcstate->generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; + gcstate->heap_size++; #endif } @@ -263,6 +265,8 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + gcstate->heap_size--; #endif } @@ -331,8 +335,9 @@ extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; // GC visit callback for tracked interpreter frames -extern int _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg); -extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); +// GH-150766: exported for greenlet +PyAPI_FUNC(int) _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg); +PyAPI_FUNC(int) _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); #ifdef Py_GIL_DISABLED extern void _PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp, diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index 2c264c39ae9de0f..c86ae242feac1ed 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -40,6 +40,8 @@ extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; +PyAPI_FUNC(PySendResult) _PyAsyncGenASend_Send(PyObject *iter, PyObject *arg, PyObject **result); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index f7d3dcd440aaf1a..1cf766ddb382dd2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1990,6 +1990,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(person)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pi_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pidfd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pointer_bits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(policy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); @@ -2031,6 +2032,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repeat)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reqrefs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(require_ready)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reserved)); @@ -2108,6 +2110,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(take_bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target_is_directory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(targetfd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(task)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tb_frame)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tb_lasti)); @@ -2150,6 +2153,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(updates)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uri)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(usedforsecurity)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(utcoffset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 22494b1798cc530..017d62e002fdff9 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -713,6 +713,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(person) STRUCT_FOR_ID(pi_factory) STRUCT_FOR_ID(pid) + STRUCT_FOR_ID(pidfd) STRUCT_FOR_ID(pointer_bits) STRUCT_FOR_ID(policy) STRUCT_FOR_ID(pos) @@ -754,6 +755,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(repeat) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) + STRUCT_FOR_ID(repr) STRUCT_FOR_ID(reqrefs) STRUCT_FOR_ID(require_ready) STRUCT_FOR_ID(reserved) @@ -831,6 +833,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(take_bytes) STRUCT_FOR_ID(target) STRUCT_FOR_ID(target_is_directory) + STRUCT_FOR_ID(targetfd) STRUCT_FOR_ID(task) STRUCT_FOR_ID(tb_frame) STRUCT_FOR_ID(tb_lasti) @@ -873,6 +876,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(updates) STRUCT_FOR_ID(uri) STRUCT_FOR_ID(usedforsecurity) + STRUCT_FOR_ID(utcoffset) STRUCT_FOR_ID(value) STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 32ed3a62b2b4a7c..a1078828afa572e 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -39,6 +39,8 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); +extern PyObject * _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index f60c5510d200752..9ed87a544234c59 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -124,7 +124,7 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); -typedef PyModuleDef_Slot *(*PyModExportFunction)(void); +typedef PySlot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 86f018e328656e2..5500c70a3b0aad9 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -191,6 +191,8 @@ struct gc_generation_stats { Py_ssize_t candidates; // Total duration of the collection in seconds: double duration; + /* heap_size on the start of the collection */ + Py_ssize_t heap_size; }; #ifdef Py_GIL_DISABLED @@ -226,7 +228,6 @@ struct _gc_runtime_state { /* linked lists of container objects */ #ifndef Py_GIL_DISABLED struct gc_generation generations[NUM_GENERATIONS]; - PyGC_Head *generation0; #else struct gc_generation young; struct gc_generation old[2]; @@ -244,6 +245,9 @@ struct _gc_runtime_state { /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + /* The number of live objects. */ + Py_ssize_t heap_size; + /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -259,16 +263,8 @@ struct _gc_runtime_state { #ifdef Py_GIL_DISABLED /* True if gc.freeze() has been used. */ int freeze_active; - - /* Memory usage of the process (RSS + swap) after last GC. */ - Py_ssize_t last_mem; - - /* This accumulates the new object count whenever collection is deferred - due to the RSS increase condition not being meet. Reset on collection. */ - Py_ssize_t deferred_count; - - /* Mutex held for gc_should_collect_mem_usage(). */ - PyMutex mutex; +#else + PyGC_Head *generation0; #endif }; @@ -278,7 +274,8 @@ struct _gc_runtime_state { { .threshold = 2000, }, \ { .threshold = 10, }, \ { .threshold = 10, }, \ - }, + }, \ + .heap_size = 0, #else #define GC_GENERATION_INIT \ .young = { .threshold = 2000, }, \ @@ -311,8 +308,6 @@ struct _import_runtime_state { Modules are added there and looked up in _imp.find_extension(). */ struct _Py_hashtable_t *hashtable; } extensions; - /* Package context -- the full module name for package imports */ - const char * pkgcontext; }; struct _import_state { @@ -352,7 +347,15 @@ struct _import_state { int lazy_imports_mode; PyObject *lazy_imports_filter; PyObject *lazy_importing_modules; + // The set stored in sys.lazy_modules if values that have been + // lazily imported. This value is only for debugging/introspection + // purposes and is not used by the runtime. PyObject *lazy_modules; + // A dict mapping package names to a set of submodule names that + // have been imported lazily from packages which have been imported + // lazily. When the package is reified we need to add a + // LazyImportObject which refers to the submodule on the module. + PyObject *lazy_pending_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; #endif @@ -827,6 +830,8 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +#define _PyInterpreterGuard_GUARDS_NOT_ALLOWED UINTPTR_MAX + /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -1002,7 +1007,7 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; - PyObject *common_consts[NUM_COMMON_CONSTANTS]; + _PyStackRef common_consts[NUM_COMMON_CONSTANTS]; bool jit; bool compiling; @@ -1053,6 +1058,11 @@ struct _is { #endif #endif + // The number of remaining finalization guards. + // If this is _PyInterpreterGuard_GUARDS_NOT_ALLOWED, then finalization + // guards can no longer be created. + uintptr_t finalization_guards; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index 28370ababc47b92..3608a7f3bce894a 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -152,6 +152,7 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; #ifdef Py_DEBUG + dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; #endif for (int i = 0; i < stacktop; i++) { @@ -207,6 +208,7 @@ _PyFrame_Initialize( frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif @@ -230,21 +232,51 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - assert(frame->stackpointer != NULL); - _PyStackRef *sp = frame->stackpointer; -#ifndef NDEBUG - frame->stackpointer = NULL; -#endif - return sp; + return frame->stackpointer; } static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - assert(frame->stackpointer == NULL); frame->stackpointer = stack_pointer; } +static inline void +_PyFrame_StackPointerValidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif + frame->stackpointer_valid = 1; +#endif +} + +static inline void +_PyFrame_StackPointerInvalidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 1); +#endif + frame->stackpointer_valid = 0; +#endif +} + +static inline void +_PyFrame_StackAssertInvalid(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif +#endif +} + + /* Determine whether a frame is incomplete. * A frame is incomplete if it is part way through * creating cell objects or a generator or coroutine. @@ -282,6 +314,20 @@ _PyThreadState_GetFrame(PyThreadState *tstate) return _PyFrame_GetFirstComplete(tstate->current_frame); } +// Update last_profiled_frame for remote profiler frame caching. +// Only update if we're removing the exact frame that was last profiled. +// This avoids corrupting the cache when transient frames (called and returned +// between profiler samples) update last_profiled_frame to addresses the +// profiler never saw. +#define _PyThreadState_UpdateLastProfiledFrame(tstate, frame, previous) \ + do { \ + PyThreadState *tstate_ = (tstate); \ + _PyInterpreterFrame *frame_ = (frame); \ + if (tstate_->last_profiled_frame == frame_) { \ + tstate_->last_profiled_frame = (previous); \ + } \ + } while (0) + /* For use by _PyFrame_GetFrameObject Do not call directly. */ PyAPI_FUNC(PyFrameObject *) @@ -392,6 +438,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif frame->return_offset = 0; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 38510685f4093cd..4d267e35504b94d 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -44,7 +44,8 @@ struct _PyInterpreterFrame { char owner; #ifdef Py_DEBUG uint8_t visited:1; - uint8_t lltrace:7; + uint8_t stackpointer_valid:1; + uint8_t lltrace:6; #else uint8_t visited; #endif diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 39c2a30f6e979de..59a7b16073f886c 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -18,8 +18,9 @@ #define INTRINSIC_TYPEVARTUPLE 9 #define INTRINSIC_SUBSCRIPT_GENERIC 10 #define INTRINSIC_TYPEALIAS 11 +#define INTRINSIC_BUILD_FROZENSET 12 -#define MAX_INTRINSIC_1 11 +#define MAX_INTRINSIC_1 12 /* Binary Functions: */ diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 2f97cc26eaf115e..9ce6ec97b6090e4 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -31,6 +31,8 @@ _Py_CODEUNIT *_PyJIT_Entry( int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr); +PyAPI_FUNC(void) _Py_jit_assert_within_stack_bounds(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno); +PyAPI_FUNC(int) _Py_jit_assertion_failure(int line); #endif // _Py_JIT diff --git a/Include/internal/pycore_jit_publish.h b/Include/internal/pycore_jit_publish.h new file mode 100644 index 000000000000000..7ee77f5c996fafe --- /dev/null +++ b/Include/internal/pycore_jit_publish.h @@ -0,0 +1,31 @@ +#ifndef Py_INTERNAL_JIT_PUBLISH_H +#define Py_INTERNAL_JIT_PUBLISH_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +typedef struct _PyJitCodeRegistration _PyJitCodeRegistration; + +#ifdef _Py_JIT + +/* Publish JIT code to optional tooling backends. + * + * The return value is a backend-specific deregistration handle, not a + * success/failure indicator. NULL means there is nothing to unregister later: + * perf does not need a handle, and GDB/GNU backtrace registration failures + * are intentionally non-fatal because tooling support must not make JIT + * compilation fail. + */ +_PyJitCodeRegistration *_PyJit_RegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJit_UnregisterCode(_PyJitCodeRegistration *registration); + +#endif // _Py_JIT + +#endif // Py_INTERNAL_JIT_PUBLISH_H diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h index 2d325ad9562286e..7099b88812ce7be 100644 --- a/Include/internal/pycore_jit_unwind.h +++ b/Include/internal/pycore_jit_unwind.h @@ -10,9 +10,15 @@ #if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__) # define PY_HAVE_JIT_GDB_UNWIND +# if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ + defined(_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION) +# define PY_HAVE_JIT_GNU_BACKTRACE_UNWIND +# endif #endif -#if defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) +#if defined(PY_HAVE_PERF_TRAMPOLINE) \ + || defined(PY_HAVE_JIT_GDB_UNWIND) \ + || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) #if defined(PY_HAVE_JIT_GDB_UNWIND) extern PyMutex _Py_jit_debug_mutex; @@ -63,6 +69,13 @@ void *_PyJitUnwind_GdbRegisterCode(const void *code_addr, void _PyJitUnwind_GdbUnregisterCode(void *handle); -#endif // defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) +#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) +void *_PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr, + size_t code_size); + +void _PyJitUnwind_GnuBacktraceUnregisterCode(void *handle); +#endif + +#endif // JIT unwind support #endif // Py_INTERNAL_JIT_UNWIND_H diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index fd918e13f2d1c72..6a8fa124ba38a7c 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -296,9 +296,12 @@ Known values: Python 3.15a8 3663 (Merge GET_ITER and GET_YIELD_FROM_ITER. Modify SEND to make it a bit more like FOR_ITER) Python 3.15a8 3664 (Fix __qualname__ for __annotate__ functions) Python 3.15a8 3665 (Add FOR_ITER_VIRTUAL and GET_ITER specializations) + Python 3.15b1 3666 (Add SEND_VIRTUAL and SEND_ASYNC_GEN specializations) + Python 3.16a0 3700 (Initial version) + Python 3.16a0 3701 (Add CONSTANT_EMPTY_TUPLE to LOAD_COMMON_CONSTANT) - Python 3.16 will start with 3700 + Python 3.17 will start with 3750 Please don't copy-paste the same pre-release tag for new entries above!!! You should always use the *upcoming* tag. For example, if 3.12a6 came out @@ -309,7 +312,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3665 +#define PYC_MAGIC_NUMBER 3701 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_mmap.h b/Include/internal/pycore_mmap.h index 897816db01077f6..c117cbd16283da9 100644 --- a/Include/internal/pycore_mmap.h +++ b/Include/internal/pycore_mmap.h @@ -11,12 +11,12 @@ extern "C" { #include "pycore_pystate.h" -#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) # include # include #endif -#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) static inline int _PyAnnotateMemoryMap(void *addr, size_t size, const char *name) { diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index c2c508c1a71c5c0..98880cd83780dca 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -266,12 +266,6 @@ _Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED; } -static inline int -_Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) -{ - return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; -} - // Merge the local and shared reference count fields and add `extra` to the // refcount when merging. Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); @@ -836,9 +830,14 @@ _PyObject_IS_GC(PyObject *obj) && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } -// Fast inlined version of PyObject_Hash() -static inline Py_hash_t -_PyObject_HashFast(PyObject *op) +// Fast inlined version of PyObject_Hash(). Dictionaries are very +// likely to include string keys (class and instance attributes, +// json, ...) so we include a fast path for strings. +// This function should not be used in a collection if str is not +// very likely, since it is slower than PyObject_Hash() on types +// other than str. See gh-137759. +static inline Py_ALWAYS_INLINE Py_hash_t +_PyObject_HashDictKey(PyObject *op) { if (PyUnicode_CheckExact(op)) { Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8c4134061de94c8..f145b7f034c52c0 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -438,8 +438,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case SEND: return 3; + case SEND_ASYNC_GEN: + return 3; case SEND_GEN: return 3; + case SEND_VIRTUAL: + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -935,8 +939,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case SEND: return 3; + case SEND_ASYNC_GEN: + return 3; case SEND_GEN: return 2; + case SEND_VIRTUAL: + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -1190,7 +1198,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_RECORDS_VALUE_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, @@ -1223,9 +1231,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1295,10 +1303,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [SEND_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1335,7 +1345,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1438,12 +1448,12 @@ _PyOpcode_macro_expansion[256] = { [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, OPARG_SIMPLE, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, OPARG_SIMPLE, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } }, - [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, + [FOR_ITER] = { .nuops = 2, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _FOR_ITER, OPARG_REPLACED, 0 } } }, [FOR_ITER_GEN] = { .nuops = 4, .uops = { { _RECORD_NOS_GEN_FUNC, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, - [FOR_ITER_VIRTUAL] = { .nuops = 2, .uops = { { _GUARD_NOS_ITER_VIRTUAL, OPARG_SIMPLE, 1 }, { _FOR_ITER_VIRTUAL, OPARG_REPLACED, 1 } } }, + [FOR_ITER_VIRTUAL] = { .nuops = 2, .uops = { { _GUARD_TOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _FOR_ITER_VIRTUAL, OPARG_REPLACED, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { _GET_ANEXT, OPARG_SIMPLE, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { _GET_AWAITABLE, OPARG_SIMPLE, 0 } } }, @@ -1466,10 +1476,10 @@ _PyOpcode_macro_expansion[256] = { [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_MODULE] = { .nuops = 4, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_PROPERTY] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 2, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_SLOT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_WITH_HINT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, @@ -1514,7 +1524,9 @@ _PyOpcode_macro_expansion[256] = { [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, + [SEND_ASYNC_GEN] = { .nuops = 2, .uops = { { _GUARD_3OS_ASYNC_GEN_ASEND, OPARG_SIMPLE, 1 }, { _SEND_ASYNC_GEN, OPARG_REPLACED, 1 } } }, [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [SEND_VIRTUAL] = { .nuops = 3, .uops = { { _GUARD_TOS_IS_NONE, OPARG_SIMPLE, 1 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _SEND_VIRTUAL, OPARG_REPLACED, 1 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, OPARG_SIMPLE, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, OPARG_SIMPLE, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, OPARG_SIMPLE, 0 } } }, @@ -1758,7 +1770,9 @@ const char *_PyOpcode_OpName[267] = { [RETURN_GENERATOR] = "RETURN_GENERATOR", [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", + [SEND_ASYNC_GEN] = "SEND_ASYNC_GEN", [SEND_GEN] = "SEND_GEN", + [SEND_VIRTUAL] = "SEND_VIRTUAL", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [SETUP_CLEANUP] = "SETUP_CLEANUP", [SETUP_FINALLY] = "SETUP_FINALLY", @@ -1810,6 +1824,7 @@ const uint8_t _PyOpcode_Caches[256] = { [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, + [FOR_ITER] = 1, [UNPACK_SEQUENCE] = 1, [STORE_ATTR] = 4, [LOAD_GLOBAL] = 4, @@ -1823,7 +1838,6 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, [GET_ITER] = 1, - [FOR_ITER] = 1, [CALL] = 3, [CALL_KW] = 3, [CALL_FUNCTION_EX] = 1, @@ -1842,8 +1856,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [217] = 217, - [218] = 218, [219] = 219, [220] = 220, [221] = 221, @@ -2052,7 +2064,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, [SEND] = SEND, + [SEND_ASYNC_GEN] = SEND, [SEND_GEN] = SEND, + [SEND_VIRTUAL] = SEND, [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, [SET_ADD] = SET_ADD, [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, @@ -2103,8 +2117,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 217: \ - case 218: \ case 219: \ case 220: \ case 221: \ diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 3e2c4ae411c925a..7067b48ec22cb3c 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -80,7 +80,9 @@ extern "C" { #define CONSTANT_TRUE 9 #define CONSTANT_FALSE 10 #define CONSTANT_MINUS_ONE 11 -#define NUM_COMMON_CONSTANTS 12 +#define CONSTANT_BUILTIN_FROZENSET 12 +#define CONSTANT_EMPTY_TUPLE 13 +#define NUM_COMMON_CONSTANTS 14 /* Values used in the oparg for RESUME */ #define RESUME_AT_FUNC_START 0 diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index a0727c045e51ec6..3d60638649dcb5a 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_typedefs.h" // _PyInterpreterFrame +#include "pycore_jit_publish.h" #include "pycore_uop.h" // _PyUOpInstruction #include "pycore_uop_ids.h" #include "pycore_stackref.h" // _PyStackRef @@ -30,9 +31,8 @@ extern "C" { * 4. A push followed by a matching return is net-zero on frame-specific * fitness, excluding per-slot costs. */ -#define MAX_TARGET_LENGTH (UOP_MAX_TRACE_LENGTH / 2) #define OPTIMIZER_EFFECTIVENESS 2 -#define FITNESS_INITIAL (MAX_TARGET_LENGTH * OPTIMIZER_EFFECTIVENESS) +#define MAX_TARGET_LENGTH (FITNESS_INITIAL / OPTIMIZER_EFFECTIVENESS) /* Exit quality thresholds: trace stops when fitness < exit_quality. * Higher = trace is more willing to stop here. */ @@ -198,7 +198,7 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; - void *jit_gdb_handle; + _PyJitCodeRegistration *jit_registration; _PyExitData exits[1]; } _PyExecutorObject; @@ -312,7 +312,7 @@ int _Py_uop_analyze_and_optimize( _PyUOpInstruction *input, int trace_len, int curr_stackentries, _PyUOpInstruction *output, _PyBloomFilter *dependencies); -extern PyTypeObject _PyUOpExecutor_Type; +PyAPI_DATA(PyTypeObject) _PyUOpExecutor_Type; #define UOP_FORMAT_TARGET 0 diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index fafdd728a8229a9..d8ec306a0dae3fc 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -138,6 +138,7 @@ extern "C" { #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else @@ -201,6 +202,7 @@ extern "C" { #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) #define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) #define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 84cb72fa6fd1b26..3056dc44cc0f1b1 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -27,14 +27,14 @@ _Py_HashPointerRaw(const void *ptr) * pppppppp ssssssss ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeeeeeee pyexpat XML hash salt + * eeeeeeee eeeeeeee eeeeeeee pyexpat XML hash salt * * memory layout on 32 bit systems * cccccccc cccccccc cccccccc uc * ppppssss ........ ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeee.... pyexpat XML hash salt + * eeeeeeee eeeeeeee eeee.... pyexpat XML hash salt * * (*) The siphash member may not be available on 32 bit platforms without * an unsigned int64 data type. @@ -58,7 +58,9 @@ typedef union { Py_hash_t suffix; } djbx33a; struct { - unsigned char padding[16]; + /* 16 bytes for XML_SetHashSalt16Bytes */ + uint8_t hashsalt16[16]; + /* 4/8 bytes for legacy XML_SetHashSalt */ Py_hash_t hashsalt; } expat; } _Py_HashSecret_t; diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 8faf7a4d403f846..12e1e78526db35a 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -18,9 +18,6 @@ extern int _Py_SetFileSystemEncoding( const char *errors); extern void _Py_ClearFileSystemEncoding(void); extern PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate); -#ifdef MS_WINDOWS -extern int _PyUnicode_EnableLegacyWindowsFSEncoding(void); -#endif extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 189a8dde9f09ed6..ca6819d2cd44730 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -266,6 +266,8 @@ extern int _PyOS_InterruptOccurred(PyThreadState *tstate); PyMutex_LockFlags(&(runtime)->interpreters.mutex, _Py_LOCK_DONT_DETACH) #define HEAD_UNLOCK(runtime) \ PyMutex_Unlock(&(runtime)->interpreters.mutex) +#define ASSERT_HEAD_IS_LOCKED(runtime) \ + assert(PyMutex_IsLocked(&(runtime)->interpreters.mutex)) #define _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) \ for (PyThreadState *t = interp->threads.head; t; t = t->next) @@ -314,15 +316,20 @@ static uintptr_t return_pointer_as_int(char* p) { static inline uintptr_t _Py_get_machine_stack_pointer(void) { -#if _Py__has_builtin(__builtin_frame_address) || defined(__GNUC__) - return (uintptr_t)__builtin_frame_address(0); -#elif defined(_MSC_VER) - return (uintptr_t)_AddressOfReturnAddress(); + uintptr_t result; +#if defined(_M_ARM64) + result = __getReg(31); +#elif defined(_M_X64) || defined(_M_IX86) + result = (uintptr_t)_AddressOfReturnAddress(); +#elif defined(__aarch64__) + __asm__ ("mov %0, sp" : "=r" (result)); +#elif defined(__x86_64__) + __asm__("{movq %%rsp, %0" : "=r" (result)); #else char here; - /* Avoid compiler warning about returning stack address */ - return return_pointer_as_int(&here); + result = (uintptr_t)&here; #endif + return result; } static inline intptr_t @@ -338,6 +345,20 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) #endif } +/* PEP 788 structures. */ + +struct PyInterpreterGuard { + PyInterpreterState *interp; +}; + +struct PyInterpreterView { + int64_t id; +}; + +// Exports for '_testinternalcapi' shared extension +PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GuardCountdown(PyInterpreterState *interp); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterGuard_GetInterpreter(PyInterpreterGuard *guard); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index 2a544edc431e6b1..66dd7cd843b04fc 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -46,7 +46,8 @@ extern PyObject * _Py_CompileStringObjectWithModule( * stack consumption of PyEval_EvalDefault */ #if (defined(Py_DEBUG) \ || defined(_Py_ADDRESS_SANITIZER) \ - || defined(_Py_THREAD_SANITIZER)) + || defined(_Py_THREAD_SANITIZER)) \ + || defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER) # define _PyOS_LOG2_STACK_MARGIN 12 #else # define _PyOS_LOG2_STACK_MARGIN 11 diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 892c3cdd9623a2f..75273243ef6df06 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1988,6 +1988,7 @@ extern "C" { INIT_ID(person), \ INIT_ID(pi_factory), \ INIT_ID(pid), \ + INIT_ID(pidfd), \ INIT_ID(pointer_bits), \ INIT_ID(policy), \ INIT_ID(pos), \ @@ -2029,6 +2030,7 @@ extern "C" { INIT_ID(repeat), \ INIT_ID(repl), \ INIT_ID(replace), \ + INIT_ID(repr), \ INIT_ID(reqrefs), \ INIT_ID(require_ready), \ INIT_ID(reserved), \ @@ -2106,6 +2108,7 @@ extern "C" { INIT_ID(take_bytes), \ INIT_ID(target), \ INIT_ID(target_is_directory), \ + INIT_ID(targetfd), \ INIT_ID(task), \ INIT_ID(tb_frame), \ INIT_ID(tb_lasti), \ @@ -2148,6 +2151,7 @@ extern "C" { INIT_ID(updates), \ INIT_ID(uri), \ INIT_ID(usedforsecurity), \ + INIT_ID(utcoffset), \ INIT_ID(value), \ INIT_ID(values), \ INIT_ID(version), \ diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 24d0135ed1aeca8..92d1a15177f79eb 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -35,6 +35,9 @@ extern void _PySet_ClearInternal(PySetObject *so); PyAPI_FUNC(int) _PySet_AddTakeRef(PySetObject *so, PyObject *key); +PyObject * +_PySet_Freeze(PyObject *set); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index ba8b1f1cb27dee3..b6c821764886c36 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -12,7 +12,7 @@ extern "C" { /* runtime lifecycle */ PyAPI_FUNC(PyObject *) -_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); +_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop, PyObject *step); #ifdef __cplusplus } diff --git a/Include/internal/pycore_slots.h b/Include/internal/pycore_slots.h new file mode 100644 index 000000000000000..960a632aea17eea --- /dev/null +++ b/Include/internal/pycore_slots.h @@ -0,0 +1,138 @@ +#ifndef _Py_PYCORE_SLOTS_H +#define _Py_PYCORE_SLOTS_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +/* Slot data type */ +typedef enum _PySlot_DTYPE { + _PySlot_DTYPE_VOID, + _PySlot_DTYPE_FUNC, + _PySlot_DTYPE_PTR, + _PySlot_DTYPE_SIZE, + _PySlot_DTYPE_INT64, + _PySlot_DTYPE_UINT64, +}_PySlot_DTYPE; + +/* Slot kind, used to identify: + * - the thing the slot initializes (type/module/special) + * - the struct type (PySlot/PyType_Slot/PyModuleDef_Slot) + */ +typedef enum _PySlot_KIND { + _PySlot_KIND_TYPE, + _PySlot_KIND_MOD, + _PySlot_KIND_COMPAT, + _PySlot_KIND_SLOT, +} _PySlot_KIND; + +typedef enum _PySlot_PROBLEM_HANDLING { + _PySlot_PROBLEM_ALLOW, + _PySlot_PROBLEM_DEPRECATED, + _PySlot_PROBLEM_REJECT, +} _PySlot_PROBLEM_HANDLING; + +PyAPI_DATA(const char *const) _PySlot_names[]; + +#define _PySlot_MAX_NESTING 5 + +/* State for one nesting level of a slots iterator */ +typedef struct _PySlotIterator_state { + union { + // tagged by slot_struct_kind: + const PySlot *slot; // with _PySlot_KIND_SLOT + const PyType_Slot *tp_slot; // with _PySlot_KIND_TYPE + const PyModuleDef_Slot *mod_slot; // with _PySlot_KIND_MOD + const void *any_slot; + }; + _PySlot_KIND slot_struct_kind; +} _PySlotIterator_state; + +#define _PySlot_SEEN_ENTRY_BITS (8 * sizeof(unsigned int)) + +/* State for a slots iterator */ +typedef struct { + _PySlotIterator_state *state; + _PySlotIterator_state states[_PySlot_MAX_NESTING]; + unsigned int seen[_Py_slot_COUNT / _PySlot_SEEN_ENTRY_BITS + 1]; + _PySlot_KIND kind; + uint8_t recursion_level; + bool is_at_end :1; + bool is_first_run :1; + + // Name of the object (type/module) being defined, NULL if unknown. + // Must be set by the callers as soon as it's known. + const char *name; + + /* Output information: */ + + // The slot. Always a copy; may be modified by caller of the iterator. + PySlot current; + +} _PySlotIterator; + +/* Initialize an iterator using a PySlot array */ +PyAPI_FUNC(void) +_PySlotIterator_Init(_PySlotIterator *it, const PySlot *slots, + _PySlot_KIND result_kind); + +/* Initialize an iterator using a legacy slot array */ +PyAPI_FUNC(void) +_PySlotIterator_InitLegacy(_PySlotIterator *it, const void *slots, + _PySlot_KIND kind); + +/* Reset a *successfully exhausted* iterator to the beginning. + * The *slots* must be the same as for the previous + * `_PySlotIterator_InitWithKind` call. + * (Unlike creating a new iterator, we can skip some validation after Rewind.) + */ +PyAPI_FUNC(void) _PySlotIterator_Rewind(_PySlotIterator *it, const void *slots); + +/* Iteration function. + * + * Return false at the end (when successfully exhausted). + * Otherwise (even on error), fill output information in `it` and return true. + * + * On error, set an exception and set `it->current.sl_id` to `Py_slot_invalid`. + */ +PyAPI_FUNC(bool) _PySlotIterator_Next(_PySlotIterator *it); + +/* Return 1 if given slot was "seen" by an earlier _PySlotIterator_Next call. + * (This state is not reset by rewinding.) + */ +PyAPI_FUNC(bool) _PySlotIterator_SawSlot(_PySlotIterator *, int); + +static inline const char * +_PySlot_GetName(uint16_t id) +{ + if (id >= _Py_slot_COUNT) { + return ""; + } + if (id == Py_slot_invalid) { + return "Py_slot_invalid"; + } + return _PySlot_names[id]; +} + +static inline void +_PySlot_err_bad_slot(char *kind, uint16_t id) +{ + if (id < _Py_slot_COUNT) { + PyErr_Format(PyExc_SystemError, "invalid %s slot %d (%s)", + kind, (unsigned int)id, _PySlot_names[id]); + } + else if (id == Py_slot_invalid) { + PyErr_Format(PyExc_SystemError, "invalid slot (Py_slot_invalid, %u)", + (unsigned int)id); + } + else { + PyErr_Format(PyExc_SystemError, "unknown %s slot ID %u", + kind, (unsigned int)id); + } +} + +#include "internal/pycore_slots_generated.h" + +#endif // _Py_PYCORE_SLOTS_H diff --git a/Include/internal/pycore_slots_generated.h b/Include/internal/pycore_slots_generated.h new file mode 100644 index 000000000000000..73a77070038cef4 --- /dev/null +++ b/Include/internal/pycore_slots_generated.h @@ -0,0 +1,958 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_INTERNAL_SLOTS_GENERATED_H +#define _PY_HAVE_INTERNAL_SLOTS_GENERATED_H + +static inline uint16_t +_PySlot_resolve_type_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return Py_bf_getbuffer; + case 2: + return Py_bf_releasebuffer; + case 3: + return Py_mp_ass_subscript; + case 4: + return Py_mp_length; + case Py_slot_end: + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_doc: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_members: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_tp_token: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + case Py_slot_subslots: + case Py_tp_slots: + case Py_tp_name: + case Py_tp_basicsize: + case Py_tp_extra_basicsize: + case Py_tp_itemsize: + case Py_tp_flags: + case Py_tp_metaclass: + case Py_tp_module: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline uint16_t +_PySlot_resolve_mod_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return Py_mod_create; + case 2: + return Py_mod_exec; + case 3: + return Py_mod_multiple_interpreters; + case 4: + return Py_mod_gil; + case Py_slot_end: + case Py_mod_create: + case Py_mod_exec: + case Py_mod_multiple_interpreters: + case Py_mod_gil: + case Py_slot_subslots: + case Py_mod_slots: + case Py_mod_name: + case Py_mod_doc: + case Py_mod_state_size: + case Py_mod_methods: + case Py_mod_state_traverse: + case Py_mod_state_clear: + case Py_mod_state_free: + case Py_mod_abi: + case Py_mod_token: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline void* +_PySlot_type_getslot(PyTypeObject *tp, uint16_t slot_id) +{ + switch (slot_id) { + case Py_mp_subscript: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_subscript; + case Py_nb_absolute: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_absolute; + case Py_nb_add: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_add; + case Py_nb_and: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_and; + case Py_nb_bool: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_bool; + case Py_nb_divmod: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_divmod; + case Py_nb_float: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_float; + case Py_nb_floor_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_floor_divide; + case Py_nb_index: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_index; + case Py_nb_inplace_add: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_add; + case Py_nb_inplace_and: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_and; + case Py_nb_inplace_floor_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_floor_divide; + case Py_nb_inplace_lshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_lshift; + case Py_nb_inplace_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_multiply; + case Py_nb_inplace_or: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_or; + case Py_nb_inplace_power: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_power; + case Py_nb_inplace_remainder: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_remainder; + case Py_nb_inplace_rshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_rshift; + case Py_nb_inplace_subtract: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_subtract; + case Py_nb_inplace_true_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_true_divide; + case Py_nb_inplace_xor: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_xor; + case Py_nb_int: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_int; + case Py_nb_invert: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_invert; + case Py_nb_lshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_lshift; + case Py_nb_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_multiply; + case Py_nb_negative: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_negative; + case Py_nb_or: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_or; + case Py_nb_positive: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_positive; + case Py_nb_power: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_power; + case Py_nb_remainder: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_remainder; + case Py_nb_rshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_rshift; + case Py_nb_subtract: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_subtract; + case Py_nb_true_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_true_divide; + case Py_nb_xor: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_xor; + case Py_sq_ass_item: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_ass_item; + case Py_sq_concat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_concat; + case Py_sq_contains: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_contains; + case Py_sq_inplace_concat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_inplace_concat; + case Py_sq_inplace_repeat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_inplace_repeat; + case Py_sq_item: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_item; + case Py_sq_length: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_length; + case Py_sq_repeat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_repeat; + case Py_tp_alloc: + return (void*)tp->tp_alloc; + case Py_tp_base: + return (void*)tp->tp_base; + case Py_tp_bases: + return (void*)tp->tp_bases; + case Py_tp_call: + return (void*)tp->tp_call; + case Py_tp_clear: + return (void*)tp->tp_clear; + case Py_tp_dealloc: + return (void*)tp->tp_dealloc; + case Py_tp_del: + return (void*)tp->tp_del; + case Py_tp_descr_get: + return (void*)tp->tp_descr_get; + case Py_tp_descr_set: + return (void*)tp->tp_descr_set; + case Py_tp_doc: + return (void*)tp->tp_doc; + case Py_tp_getattr: + return (void*)tp->tp_getattr; + case Py_tp_getattro: + return (void*)tp->tp_getattro; + case Py_tp_hash: + return (void*)tp->tp_hash; + case Py_tp_init: + return (void*)tp->tp_init; + case Py_tp_is_gc: + return (void*)tp->tp_is_gc; + case Py_tp_iter: + return (void*)tp->tp_iter; + case Py_tp_iternext: + return (void*)tp->tp_iternext; + case Py_tp_methods: + return (void*)tp->tp_methods; + case Py_tp_new: + return (void*)tp->tp_new; + case Py_tp_repr: + return (void*)tp->tp_repr; + case Py_tp_richcompare: + return (void*)tp->tp_richcompare; + case Py_tp_setattr: + return (void*)tp->tp_setattr; + case Py_tp_setattro: + return (void*)tp->tp_setattro; + case Py_tp_str: + return (void*)tp->tp_str; + case Py_tp_traverse: + return (void*)tp->tp_traverse; + case Py_tp_members: + return (void*)tp->tp_members; + case Py_tp_getset: + return (void*)tp->tp_getset; + case Py_tp_free: + return (void*)tp->tp_free; + case Py_nb_matrix_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_matrix_multiply; + case Py_nb_inplace_matrix_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_matrix_multiply; + case Py_am_await: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_await; + case Py_am_aiter: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_aiter; + case Py_am_anext: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_anext; + case Py_tp_finalize: + return (void*)tp->tp_finalize; + case Py_am_send: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_send; + case Py_tp_vectorcall: + return (void*)tp->tp_vectorcall; + case Py_tp_token: + if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) return NULL; + return (void*)((PyHeapTypeObject*)tp)->ht_token; + case Py_bf_getbuffer: + if (!(tp->tp_as_buffer)) return NULL; + return (void*)tp->tp_as_buffer->bf_getbuffer; + case Py_bf_releasebuffer: + if (!(tp->tp_as_buffer)) return NULL; + return (void*)tp->tp_as_buffer->bf_releasebuffer; + case Py_mp_ass_subscript: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_ass_subscript; + case Py_mp_length: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_length; + } + _PySlot_err_bad_slot("PyType_GetSlot", slot_id); + return NULL; +} + +static inline void +_PySlot_heaptype_apply_field_slot(PyHeapTypeObject *ht, PySlot slot) +{ + switch (slot.sl_id) { + case Py_mp_subscript: + ht->as_mapping.mp_subscript = (binaryfunc)slot.sl_func; + break; + case Py_nb_absolute: + ht->as_number.nb_absolute = (unaryfunc)slot.sl_func; + break; + case Py_nb_add: + ht->as_number.nb_add = (binaryfunc)slot.sl_func; + break; + case Py_nb_and: + ht->as_number.nb_and = (binaryfunc)slot.sl_func; + break; + case Py_nb_bool: + ht->as_number.nb_bool = (inquiry)slot.sl_func; + break; + case Py_nb_divmod: + ht->as_number.nb_divmod = (binaryfunc)slot.sl_func; + break; + case Py_nb_float: + ht->as_number.nb_float = (unaryfunc)slot.sl_func; + break; + case Py_nb_floor_divide: + ht->as_number.nb_floor_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_index: + ht->as_number.nb_index = (unaryfunc)slot.sl_func; + break; + case Py_nb_inplace_add: + ht->as_number.nb_inplace_add = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_and: + ht->as_number.nb_inplace_and = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_floor_divide: + ht->as_number.nb_inplace_floor_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_lshift: + ht->as_number.nb_inplace_lshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_multiply: + ht->as_number.nb_inplace_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_or: + ht->as_number.nb_inplace_or = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_power: + ht->as_number.nb_inplace_power = (ternaryfunc)slot.sl_func; + break; + case Py_nb_inplace_remainder: + ht->as_number.nb_inplace_remainder = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_rshift: + ht->as_number.nb_inplace_rshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_subtract: + ht->as_number.nb_inplace_subtract = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_true_divide: + ht->as_number.nb_inplace_true_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_xor: + ht->as_number.nb_inplace_xor = (binaryfunc)slot.sl_func; + break; + case Py_nb_int: + ht->as_number.nb_int = (unaryfunc)slot.sl_func; + break; + case Py_nb_invert: + ht->as_number.nb_invert = (unaryfunc)slot.sl_func; + break; + case Py_nb_lshift: + ht->as_number.nb_lshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_multiply: + ht->as_number.nb_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_negative: + ht->as_number.nb_negative = (unaryfunc)slot.sl_func; + break; + case Py_nb_or: + ht->as_number.nb_or = (binaryfunc)slot.sl_func; + break; + case Py_nb_positive: + ht->as_number.nb_positive = (unaryfunc)slot.sl_func; + break; + case Py_nb_power: + ht->as_number.nb_power = (ternaryfunc)slot.sl_func; + break; + case Py_nb_remainder: + ht->as_number.nb_remainder = (binaryfunc)slot.sl_func; + break; + case Py_nb_rshift: + ht->as_number.nb_rshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_subtract: + ht->as_number.nb_subtract = (binaryfunc)slot.sl_func; + break; + case Py_nb_true_divide: + ht->as_number.nb_true_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_xor: + ht->as_number.nb_xor = (binaryfunc)slot.sl_func; + break; + case Py_sq_ass_item: + ht->as_sequence.sq_ass_item = (ssizeobjargproc)slot.sl_func; + break; + case Py_sq_concat: + ht->as_sequence.sq_concat = (binaryfunc)slot.sl_func; + break; + case Py_sq_contains: + ht->as_sequence.sq_contains = (objobjproc)slot.sl_func; + break; + case Py_sq_inplace_concat: + ht->as_sequence.sq_inplace_concat = (binaryfunc)slot.sl_func; + break; + case Py_sq_inplace_repeat: + ht->as_sequence.sq_inplace_repeat = (ssizeargfunc)slot.sl_func; + break; + case Py_sq_item: + ht->as_sequence.sq_item = (ssizeargfunc)slot.sl_func; + break; + case Py_sq_length: + ht->as_sequence.sq_length = (lenfunc)slot.sl_func; + break; + case Py_sq_repeat: + ht->as_sequence.sq_repeat = (ssizeargfunc)slot.sl_func; + break; + case Py_tp_alloc: + ht->ht_type.tp_alloc = (allocfunc)slot.sl_func; + break; + case Py_tp_base: + ht->ht_type.tp_base = slot.sl_ptr; + break; + case Py_tp_bases: + ht->ht_type.tp_bases = slot.sl_ptr; + break; + case Py_tp_call: + ht->ht_type.tp_call = (ternaryfunc)slot.sl_func; + break; + case Py_tp_clear: + ht->ht_type.tp_clear = (inquiry)slot.sl_func; + break; + case Py_tp_dealloc: + ht->ht_type.tp_dealloc = (destructor)slot.sl_func; + break; + case Py_tp_del: + ht->ht_type.tp_del = (destructor)slot.sl_func; + break; + case Py_tp_descr_get: + ht->ht_type.tp_descr_get = (descrgetfunc)slot.sl_func; + break; + case Py_tp_descr_set: + ht->ht_type.tp_descr_set = (descrsetfunc)slot.sl_func; + break; + case Py_tp_doc: + ht->ht_type.tp_doc = slot.sl_ptr; + break; + case Py_tp_getattr: + ht->ht_type.tp_getattr = (getattrfunc)slot.sl_func; + break; + case Py_tp_getattro: + ht->ht_type.tp_getattro = (getattrofunc)slot.sl_func; + break; + case Py_tp_hash: + ht->ht_type.tp_hash = (hashfunc)slot.sl_func; + break; + case Py_tp_init: + ht->ht_type.tp_init = (initproc)slot.sl_func; + break; + case Py_tp_is_gc: + ht->ht_type.tp_is_gc = (inquiry)slot.sl_func; + break; + case Py_tp_iter: + ht->ht_type.tp_iter = (getiterfunc)slot.sl_func; + break; + case Py_tp_iternext: + ht->ht_type.tp_iternext = (iternextfunc)slot.sl_func; + break; + case Py_tp_methods: + ht->ht_type.tp_methods = slot.sl_ptr; + break; + case Py_tp_new: + ht->ht_type.tp_new = (newfunc)slot.sl_func; + break; + case Py_tp_repr: + ht->ht_type.tp_repr = (reprfunc)slot.sl_func; + break; + case Py_tp_richcompare: + ht->ht_type.tp_richcompare = (richcmpfunc)slot.sl_func; + break; + case Py_tp_setattr: + ht->ht_type.tp_setattr = (setattrfunc)slot.sl_func; + break; + case Py_tp_setattro: + ht->ht_type.tp_setattro = (setattrofunc)slot.sl_func; + break; + case Py_tp_str: + ht->ht_type.tp_str = (reprfunc)slot.sl_func; + break; + case Py_tp_traverse: + ht->ht_type.tp_traverse = (traverseproc)slot.sl_func; + break; + case Py_tp_members: + ht->ht_type.tp_members = slot.sl_ptr; + break; + case Py_tp_getset: + ht->ht_type.tp_getset = slot.sl_ptr; + break; + case Py_tp_free: + ht->ht_type.tp_free = (freefunc)slot.sl_func; + break; + case Py_nb_matrix_multiply: + ht->as_number.nb_matrix_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_matrix_multiply: + ht->as_number.nb_inplace_matrix_multiply = (binaryfunc)slot.sl_func; + break; + case Py_am_await: + ht->as_async.am_await = (unaryfunc)slot.sl_func; + break; + case Py_am_aiter: + ht->as_async.am_aiter = (unaryfunc)slot.sl_func; + break; + case Py_am_anext: + ht->as_async.am_anext = (unaryfunc)slot.sl_func; + break; + case Py_tp_finalize: + ht->ht_type.tp_finalize = (destructor)slot.sl_func; + break; + case Py_am_send: + ht->as_async.am_send = (sendfunc)slot.sl_func; + break; + case Py_tp_vectorcall: + ht->ht_type.tp_vectorcall = (vectorcallfunc)slot.sl_func; + break; + case Py_bf_getbuffer: + ht->as_buffer.bf_getbuffer = (getbufferproc)slot.sl_func; + break; + case Py_bf_releasebuffer: + ht->as_buffer.bf_releasebuffer = (releasebufferproc)slot.sl_func; + break; + case Py_mp_ass_subscript: + ht->as_mapping.mp_ass_subscript = (objobjargproc)slot.sl_func; + break; + case Py_mp_length: + ht->as_mapping.mp_length = (lenfunc)slot.sl_func; + break; + } +} + +static inline _PySlot_DTYPE +_PySlot_get_dtype(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: return _PySlot_DTYPE_VOID; + case Py_mp_subscript: return _PySlot_DTYPE_FUNC; + case Py_nb_absolute: return _PySlot_DTYPE_FUNC; + case Py_nb_add: return _PySlot_DTYPE_FUNC; + case Py_nb_and: return _PySlot_DTYPE_FUNC; + case Py_nb_bool: return _PySlot_DTYPE_FUNC; + case Py_nb_divmod: return _PySlot_DTYPE_FUNC; + case Py_nb_float: return _PySlot_DTYPE_FUNC; + case Py_nb_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_index: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_add: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_and: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_or: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_power: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_xor: return _PySlot_DTYPE_FUNC; + case Py_nb_int: return _PySlot_DTYPE_FUNC; + case Py_nb_invert: return _PySlot_DTYPE_FUNC; + case Py_nb_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_negative: return _PySlot_DTYPE_FUNC; + case Py_nb_or: return _PySlot_DTYPE_FUNC; + case Py_nb_positive: return _PySlot_DTYPE_FUNC; + case Py_nb_power: return _PySlot_DTYPE_FUNC; + case Py_nb_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_xor: return _PySlot_DTYPE_FUNC; + case Py_sq_ass_item: return _PySlot_DTYPE_FUNC; + case Py_sq_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_contains: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_repeat: return _PySlot_DTYPE_FUNC; + case Py_sq_item: return _PySlot_DTYPE_FUNC; + case Py_sq_length: return _PySlot_DTYPE_FUNC; + case Py_sq_repeat: return _PySlot_DTYPE_FUNC; + case Py_tp_alloc: return _PySlot_DTYPE_FUNC; + case Py_tp_base: return _PySlot_DTYPE_PTR; + case Py_tp_bases: return _PySlot_DTYPE_PTR; + case Py_tp_call: return _PySlot_DTYPE_FUNC; + case Py_tp_clear: return _PySlot_DTYPE_FUNC; + case Py_tp_dealloc: return _PySlot_DTYPE_FUNC; + case Py_tp_del: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_get: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_set: return _PySlot_DTYPE_FUNC; + case Py_tp_doc: return _PySlot_DTYPE_PTR; + case Py_tp_getattr: return _PySlot_DTYPE_FUNC; + case Py_tp_getattro: return _PySlot_DTYPE_FUNC; + case Py_tp_hash: return _PySlot_DTYPE_FUNC; + case Py_tp_init: return _PySlot_DTYPE_FUNC; + case Py_tp_is_gc: return _PySlot_DTYPE_FUNC; + case Py_tp_iter: return _PySlot_DTYPE_FUNC; + case Py_tp_iternext: return _PySlot_DTYPE_FUNC; + case Py_tp_methods: return _PySlot_DTYPE_PTR; + case Py_tp_new: return _PySlot_DTYPE_FUNC; + case Py_tp_repr: return _PySlot_DTYPE_FUNC; + case Py_tp_richcompare: return _PySlot_DTYPE_FUNC; + case Py_tp_setattr: return _PySlot_DTYPE_FUNC; + case Py_tp_setattro: return _PySlot_DTYPE_FUNC; + case Py_tp_str: return _PySlot_DTYPE_FUNC; + case Py_tp_traverse: return _PySlot_DTYPE_FUNC; + case Py_tp_members: return _PySlot_DTYPE_PTR; + case Py_tp_getset: return _PySlot_DTYPE_PTR; + case Py_tp_free: return _PySlot_DTYPE_FUNC; + case Py_nb_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_am_await: return _PySlot_DTYPE_FUNC; + case Py_am_aiter: return _PySlot_DTYPE_FUNC; + case Py_am_anext: return _PySlot_DTYPE_FUNC; + case Py_tp_finalize: return _PySlot_DTYPE_FUNC; + case Py_am_send: return _PySlot_DTYPE_FUNC; + case Py_tp_vectorcall: return _PySlot_DTYPE_FUNC; + case Py_tp_token: return _PySlot_DTYPE_PTR; + case Py_mod_create: return _PySlot_DTYPE_FUNC; + case Py_mod_exec: return _PySlot_DTYPE_FUNC; + case Py_mod_multiple_interpreters: return _PySlot_DTYPE_UINT64; + case Py_mod_gil: return _PySlot_DTYPE_UINT64; + case Py_bf_getbuffer: return _PySlot_DTYPE_FUNC; + case Py_bf_releasebuffer: return _PySlot_DTYPE_FUNC; + case Py_mp_ass_subscript: return _PySlot_DTYPE_FUNC; + case Py_mp_length: return _PySlot_DTYPE_FUNC; + case Py_slot_subslots: return _PySlot_DTYPE_PTR; + case Py_tp_slots: return _PySlot_DTYPE_PTR; + case Py_mod_slots: return _PySlot_DTYPE_PTR; + case Py_tp_name: return _PySlot_DTYPE_PTR; + case Py_tp_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_extra_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_itemsize: return _PySlot_DTYPE_SIZE; + case Py_tp_flags: return _PySlot_DTYPE_UINT64; + case Py_mod_name: return _PySlot_DTYPE_PTR; + case Py_mod_doc: return _PySlot_DTYPE_PTR; + case Py_mod_state_size: return _PySlot_DTYPE_SIZE; + case Py_mod_methods: return _PySlot_DTYPE_PTR; + case Py_mod_state_traverse: return _PySlot_DTYPE_FUNC; + case Py_mod_state_clear: return _PySlot_DTYPE_FUNC; + case Py_mod_state_free: return _PySlot_DTYPE_FUNC; + case Py_tp_metaclass: return _PySlot_DTYPE_PTR; + case Py_tp_module: return _PySlot_DTYPE_PTR; + case Py_mod_abi: return _PySlot_DTYPE_PTR; + case Py_mod_token: return _PySlot_DTYPE_PTR; + default: return _PySlot_DTYPE_VOID; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_duplicate_handling(uint16_t slot_id) +{ + switch (slot_id) { + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_tp_token: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + return _PySlot_PROBLEM_DEPRECATED; + case Py_mod_exec: + case Py_mod_abi: + return _PySlot_PROBLEM_ALLOW; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_null_handling(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: + case Py_tp_doc: + case Py_tp_token: + case Py_mod_multiple_interpreters: + case Py_mod_gil: + case Py_slot_subslots: + case Py_tp_slots: + case Py_mod_slots: + case Py_tp_basicsize: + case Py_tp_extra_basicsize: + case Py_tp_itemsize: + case Py_tp_flags: + case Py_mod_state_size: + return _PySlot_PROBLEM_ALLOW; + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_mod_create: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + return _PySlot_PROBLEM_DEPRECATED; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline bool +_PySlot_get_must_be_static(uint16_t slot_id) +{ + switch (slot_id) { + case Py_tp_methods: return true; + case Py_tp_members: return true; + case Py_tp_getset: return true; + case Py_mod_methods: return true; + } + return false; +} + +#endif /* _PY_HAVE_INTERNAL_SLOTS_GENERATED_H */ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index ca4a7c216eda532..9495ccc8ac38896 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -263,6 +263,18 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) +static inline _PyStackRef +_PyStackRef_DupImmortal(_PyStackRef ref, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + PyObject *obj = _Py_stackref_get_object(ref); + assert(_Py_IsImmortal(obj)); + return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); +} +#define PyStackRef_DupImmortal(REF) _PyStackRef_DupImmortal((REF), __FILE__, __LINE__) + static inline void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber) { @@ -633,6 +645,15 @@ PyStackRef_DUP(_PyStackRef ref) } #endif +static inline _PyStackRef +PyStackRef_DupImmortal(_PyStackRef ref) +{ + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + assert(_Py_IsImmortal(BITS_TO_PTR_MASKED(ref))); + return ref; +} + static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index fbf6bc2c41f51de..e016afaa5c56879 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -14,56 +14,6 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, P // Export for 'pyexact' shared extension PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); -/* Write the Python traceback into the file 'fd'. For example: - - Traceback (most recent call first): - File "xxx", line xxx in - File "xxx", line xxx in - ... - File "xxx", line xxx in - - This function is written for debug purpose only, to dump the traceback in - the worst case: after a segmentation fault, at fatal error, etc. That's why, - it is very limited. Strings are truncated to 100 characters and encoded to - ASCII with backslashreplace. It doesn't write the source code, only the - function name, filename and line number of each frame. Write only the first - 100 frames: if the traceback is truncated, write the line " ...". - - This function is signal safe. */ - -extern void _Py_DumpTraceback( - int fd, - PyThreadState *tstate); - -/* Write the traceback of all threads into the file 'fd'. current_thread can be - NULL. - - Return NULL on success, or an error message on error. - - This function is written for debug purpose only. It calls - _Py_DumpTraceback() for each thread, and so has the same limitations. It - only write the traceback of the first 100 threads: write "..." if there are - more threads. - - If current_tstate is NULL, the function tries to get the Python thread state - of the current thread. It is not an error if the function is unable to get - the current Python thread state. - - If interp is NULL, the function tries to get the interpreter state from - the current Python thread state, or from - _PyGILState_GetInterpreterStateUnsafe() in last resort. - - It is better to pass NULL to interp and current_tstate, the function tries - different options to retrieve this information. - - This function is signal safe. */ - -extern const char* _Py_DumpTracebackThreads( - int fd, - PyInterpreterState *interp, - PyThreadState *current_tstate, - Py_ssize_t max_threads); - /* Write a Unicode object into the file descriptor fd. Encode the string to ASCII using the backslashreplace error handler. diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index bf80f96396ea4a8..e294b16b3df60f6 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -29,6 +29,7 @@ PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRe PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); PyAPI_FUNC(PyObject *) _PyTuple_BinarySlice(PyObject *, PyObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyTuple_Concat(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_Repeat(PyObject *self, Py_ssize_t n); PyAPI_FUNC(PyObject *) _PyTuple_FromPair(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyTuple_FromPairSteal(PyObject *, PyObject *); diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 8d48cf6605ca7e3..785b77d3e3be81e 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -122,6 +122,8 @@ extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +extern int _PyType_HasSlotTpIternext(PyTypeObject *type); + extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 74d84052a2bb2be..75d5068f815b918 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -33,6 +33,7 @@ extern PyObject* _PyUnicode_ResizeCompact( Py_ssize_t length); extern PyObject* _PyUnicode_GetEmpty(void); PyAPI_FUNC(PyObject*) _PyUnicode_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyUnicode_Repeat(PyObject *str, Py_ssize_t len); /* Generic helper macro to convert characters of different types. diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index f0fc3c4f5b09006..164d9d412ef5e95 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -2632,6 +2632,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pidfd); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pointer_bits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2796,6 +2800,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(repr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reqrefs); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3104,6 +3112,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(targetfd); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(task); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3272,6 +3284,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(utcoffset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(value); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 320508e8b7a95e9..9ff87faec40d0b4 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -36,13 +36,12 @@ typedef struct _PyUOpInstruction{ #endif } _PyUOpInstruction; -// This is the length of the trace we translate initially. -#if defined(Py_DEBUG) && defined(_Py_JIT) - // With asserts, the stencils are a lot larger -#define UOP_MAX_TRACE_LENGTH 1000 -#else -#define UOP_MAX_TRACE_LENGTH 2500 -#endif +// Fitness is the target length of the trace we translate initially. The uop +// buffer has a small amount of extra space for entry/loop-closing overhead. +#define FITNESS_INITIAL 2500 + +#define UOP_TRACE_BUFFER_OVERHEAD 10 +#define UOP_MAX_TRACE_LENGTH (FITNESS_INITIAL + UOP_TRACE_BUFFER_OVERHEAD) /* Bloom filter with m = 256 * https://en.wikipedia.org/wiki/Bloom_filter */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index d1919b9f8df362f..9d28c8590afcd7b 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -153,52 +153,52 @@ extern "C" { #define _GET_ITER 415 #define _GET_ITER_TRAD 416 #define _GET_LEN GET_LEN -#define _GUARD_BINARY_OP_EXTEND 417 -#define _GUARD_BINARY_OP_EXTEND_LHS 418 -#define _GUARD_BINARY_OP_EXTEND_RHS 419 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 420 -#define _GUARD_BIT_IS_SET_POP 421 -#define _GUARD_BIT_IS_SET_POP_4 422 -#define _GUARD_BIT_IS_SET_POP_5 423 -#define _GUARD_BIT_IS_SET_POP_6 424 -#define _GUARD_BIT_IS_SET_POP_7 425 -#define _GUARD_BIT_IS_UNSET_POP 426 -#define _GUARD_BIT_IS_UNSET_POP_4 427 -#define _GUARD_BIT_IS_UNSET_POP_5 428 -#define _GUARD_BIT_IS_UNSET_POP_6 429 -#define _GUARD_BIT_IS_UNSET_POP_7 430 -#define _GUARD_CALLABLE_BUILTIN_CLASS 431 -#define _GUARD_CALLABLE_BUILTIN_FAST 432 -#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS 433 -#define _GUARD_CALLABLE_BUILTIN_O 434 -#define _GUARD_CALLABLE_ISINSTANCE 435 -#define _GUARD_CALLABLE_LEN 436 -#define _GUARD_CALLABLE_LIST_APPEND 437 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST 438 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 439 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS 440 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O 441 -#define _GUARD_CALLABLE_STR_1 442 -#define _GUARD_CALLABLE_TUPLE_1 443 -#define _GUARD_CALLABLE_TYPE_1 444 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR 445 -#define _GUARD_CODE_VERSION_RETURN_VALUE 446 -#define _GUARD_CODE_VERSION_YIELD_VALUE 447 -#define _GUARD_CODE_VERSION__PUSH_FRAME 448 -#define _GUARD_DORV_NO_DICT 449 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 450 -#define _GUARD_GLOBALS_VERSION 451 -#define _GUARD_IP_RETURN_GENERATOR 452 -#define _GUARD_IP_RETURN_VALUE 453 -#define _GUARD_IP_YIELD_VALUE 454 -#define _GUARD_IP__PUSH_FRAME 455 -#define _GUARD_IS_FALSE_POP 456 -#define _GUARD_IS_NONE_POP 457 -#define _GUARD_IS_NOT_NONE_POP 458 -#define _GUARD_IS_TRUE_POP 459 -#define _GUARD_ITERATOR 460 -#define _GUARD_ITER_VIRTUAL 461 -#define _GUARD_KEYS_VERSION 462 +#define _GUARD_3OS_ASYNC_GEN_ASEND 417 +#define _GUARD_BINARY_OP_EXTEND 418 +#define _GUARD_BINARY_OP_EXTEND_LHS 419 +#define _GUARD_BINARY_OP_EXTEND_RHS 420 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 421 +#define _GUARD_BIT_IS_SET_POP 422 +#define _GUARD_BIT_IS_SET_POP_4 423 +#define _GUARD_BIT_IS_SET_POP_5 424 +#define _GUARD_BIT_IS_SET_POP_6 425 +#define _GUARD_BIT_IS_SET_POP_7 426 +#define _GUARD_BIT_IS_UNSET_POP 427 +#define _GUARD_BIT_IS_UNSET_POP_4 428 +#define _GUARD_BIT_IS_UNSET_POP_5 429 +#define _GUARD_BIT_IS_UNSET_POP_6 430 +#define _GUARD_BIT_IS_UNSET_POP_7 431 +#define _GUARD_CALLABLE_BUILTIN_CLASS 432 +#define _GUARD_CALLABLE_BUILTIN_FAST 433 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS 434 +#define _GUARD_CALLABLE_BUILTIN_O 435 +#define _GUARD_CALLABLE_ISINSTANCE 436 +#define _GUARD_CALLABLE_LEN 437 +#define _GUARD_CALLABLE_LIST_APPEND 438 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST 439 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 440 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS 441 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O 442 +#define _GUARD_CALLABLE_STR_1 443 +#define _GUARD_CALLABLE_TUPLE_1 444 +#define _GUARD_CALLABLE_TYPE_1 445 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR 446 +#define _GUARD_CODE_VERSION_RETURN_VALUE 447 +#define _GUARD_CODE_VERSION_YIELD_VALUE 448 +#define _GUARD_CODE_VERSION__PUSH_FRAME 449 +#define _GUARD_DORV_NO_DICT 450 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 451 +#define _GUARD_GLOBALS_VERSION 452 +#define _GUARD_IP_RETURN_GENERATOR 453 +#define _GUARD_IP_RETURN_VALUE 454 +#define _GUARD_IP_YIELD_VALUE 455 +#define _GUARD_IP__PUSH_FRAME 456 +#define _GUARD_IS_FALSE_POP 457 +#define _GUARD_IS_NONE_POP 458 +#define _GUARD_IS_NOT_NONE_POP 459 +#define _GUARD_IS_TRUE_POP 460 +#define _GUARD_ITERATOR 461 +#define _GUARD_ITER_VIRTUAL 462 #define _GUARD_LOAD_SUPER_ATTR_METHOD 463 #define _GUARD_NOS_COMPACT_ASCII 464 #define _GUARD_NOS_DICT_STORE_SUBSCRIPT 465 @@ -224,26 +224,29 @@ extern "C" { #define _GUARD_TOS_FROZENDICT 485 #define _GUARD_TOS_FROZENSET 486 #define _GUARD_TOS_INT 487 -#define _GUARD_TOS_LIST 488 -#define _GUARD_TOS_OVERFLOWED 489 -#define _GUARD_TOS_SET 490 -#define _GUARD_TOS_SLICE 491 -#define _GUARD_TOS_TUPLE 492 -#define _GUARD_TOS_UNICODE 493 -#define _GUARD_TYPE 494 -#define _GUARD_TYPE_VERSION 495 -#define _GUARD_TYPE_VERSION_LOCKED 496 -#define _HANDLE_PENDING_AND_DEOPT 497 +#define _GUARD_TOS_IS_NONE 488 +#define _GUARD_TOS_LIST 489 +#define _GUARD_TOS_NOT_NULL 490 +#define _GUARD_TOS_OVERFLOWED 491 +#define _GUARD_TOS_SET 492 +#define _GUARD_TOS_SLICE 493 +#define _GUARD_TOS_TUPLE 494 +#define _GUARD_TOS_UNICODE 495 +#define _GUARD_TYPE 496 +#define _GUARD_TYPE_ITER 497 +#define _GUARD_TYPE_VERSION 498 +#define _GUARD_TYPE_VERSION_LOCKED 499 +#define _HANDLE_PENDING_AND_DEOPT 500 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 498 -#define _INIT_CALL_PY_EXACT_ARGS 499 -#define _INIT_CALL_PY_EXACT_ARGS_0 500 -#define _INIT_CALL_PY_EXACT_ARGS_1 501 -#define _INIT_CALL_PY_EXACT_ARGS_2 502 -#define _INIT_CALL_PY_EXACT_ARGS_3 503 -#define _INIT_CALL_PY_EXACT_ARGS_4 504 -#define _INSERT_NULL 505 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 501 +#define _INIT_CALL_PY_EXACT_ARGS 502 +#define _INIT_CALL_PY_EXACT_ARGS_0 503 +#define _INIT_CALL_PY_EXACT_ARGS_1 504 +#define _INIT_CALL_PY_EXACT_ARGS_2 505 +#define _INIT_CALL_PY_EXACT_ARGS_3 506 +#define _INIT_CALL_PY_EXACT_ARGS_4 507 +#define _INSERT_NULL 508 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -253,1167 +256,1190 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 506 -#define _IS_OP 507 -#define _ITER_CHECK_LIST 508 -#define _ITER_CHECK_RANGE 509 -#define _ITER_CHECK_TUPLE 510 -#define _ITER_JUMP_LIST 511 -#define _ITER_JUMP_RANGE 512 -#define _ITER_JUMP_TUPLE 513 -#define _ITER_NEXT_LIST 514 -#define _ITER_NEXT_LIST_TIER_TWO 515 -#define _ITER_NEXT_RANGE 516 -#define _ITER_NEXT_TUPLE 517 +#define _IS_NONE 509 +#define _IS_OP 510 +#define _ITER_CHECK_LIST 511 +#define _ITER_CHECK_RANGE 512 +#define _ITER_CHECK_TUPLE 513 +#define _ITER_JUMP_LIST 514 +#define _ITER_JUMP_RANGE 515 +#define _ITER_JUMP_TUPLE 516 +#define _ITER_NEXT_INLINE 517 +#define _ITER_NEXT_LIST 518 +#define _ITER_NEXT_LIST_TIER_TWO 519 +#define _ITER_NEXT_RANGE 520 +#define _ITER_NEXT_TUPLE 521 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 518 +#define _JUMP_TO_TOP 522 #define _LIST_APPEND LIST_APPEND -#define _LIST_EXTEND 519 -#define _LOAD_ATTR 520 -#define _LOAD_ATTR_CLASS 521 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME 522 -#define _LOAD_ATTR_INSTANCE_VALUE 523 -#define _LOAD_ATTR_METHOD_LAZY_DICT 524 -#define _LOAD_ATTR_METHOD_NO_DICT 525 -#define _LOAD_ATTR_METHOD_WITH_VALUES 526 -#define _LOAD_ATTR_MODULE 527 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 528 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 529 -#define _LOAD_ATTR_PROPERTY_FRAME 530 -#define _LOAD_ATTR_SLOT 531 -#define _LOAD_ATTR_WITH_HINT 532 +#define _LIST_EXTEND 523 +#define _LOAD_ATTR 524 +#define _LOAD_ATTR_CLASS 525 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME 526 +#define _LOAD_ATTR_INSTANCE_VALUE 527 +#define _LOAD_ATTR_METHOD_LAZY_DICT 528 +#define _LOAD_ATTR_METHOD_NO_DICT 529 +#define _LOAD_ATTR_METHOD_WITH_VALUES 530 +#define _LOAD_ATTR_MODULE 531 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 532 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 533 +#define _LOAD_ATTR_PROPERTY_FRAME 534 +#define _LOAD_ATTR_SLOT 535 +#define _LOAD_ATTR_WITH_HINT 536 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 533 +#define _LOAD_BYTECODE 537 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 534 -#define _LOAD_CONST_INLINE_BORROW 535 +#define _LOAD_CONST_INLINE 538 +#define _LOAD_CONST_INLINE_BORROW 539 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 536 -#define _LOAD_FAST_0 537 -#define _LOAD_FAST_1 538 -#define _LOAD_FAST_2 539 -#define _LOAD_FAST_3 540 -#define _LOAD_FAST_4 541 -#define _LOAD_FAST_5 542 -#define _LOAD_FAST_6 543 -#define _LOAD_FAST_7 544 +#define _LOAD_FAST 540 +#define _LOAD_FAST_0 541 +#define _LOAD_FAST_1 542 +#define _LOAD_FAST_2 543 +#define _LOAD_FAST_3 544 +#define _LOAD_FAST_4 545 +#define _LOAD_FAST_5 546 +#define _LOAD_FAST_6 547 +#define _LOAD_FAST_7 548 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 545 -#define _LOAD_FAST_BORROW_0 546 -#define _LOAD_FAST_BORROW_1 547 -#define _LOAD_FAST_BORROW_2 548 -#define _LOAD_FAST_BORROW_3 549 -#define _LOAD_FAST_BORROW_4 550 -#define _LOAD_FAST_BORROW_5 551 -#define _LOAD_FAST_BORROW_6 552 -#define _LOAD_FAST_BORROW_7 553 +#define _LOAD_FAST_BORROW 549 +#define _LOAD_FAST_BORROW_0 550 +#define _LOAD_FAST_BORROW_1 551 +#define _LOAD_FAST_BORROW_2 552 +#define _LOAD_FAST_BORROW_3 553 +#define _LOAD_FAST_BORROW_4 554 +#define _LOAD_FAST_BORROW_5 555 +#define _LOAD_FAST_BORROW_6 556 +#define _LOAD_FAST_BORROW_7 557 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 554 -#define _LOAD_GLOBAL_BUILTINS 555 -#define _LOAD_GLOBAL_MODULE 556 +#define _LOAD_GLOBAL 558 +#define _LOAD_GLOBAL_BUILTINS 559 +#define _LOAD_GLOBAL_MODULE 560 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 557 -#define _LOAD_SMALL_INT_0 558 -#define _LOAD_SMALL_INT_1 559 -#define _LOAD_SMALL_INT_2 560 -#define _LOAD_SMALL_INT_3 561 -#define _LOAD_SPECIAL 562 +#define _LOAD_SMALL_INT 561 +#define _LOAD_SMALL_INT_0 562 +#define _LOAD_SMALL_INT_1 563 +#define _LOAD_SMALL_INT_2 564 +#define _LOAD_SMALL_INT_3 565 +#define _LOAD_SPECIAL 566 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR -#define _LOAD_SUPER_ATTR_METHOD 563 -#define _LOCK_OBJECT 564 -#define _MAKE_CALLARGS_A_TUPLE 565 +#define _LOAD_SUPER_ATTR_METHOD 567 +#define _LOCK_OBJECT 568 +#define _MAKE_CALLARGS_A_TUPLE 569 #define _MAKE_CELL MAKE_CELL -#define _MAKE_FUNCTION 566 -#define _MAKE_HEAP_SAFE 567 -#define _MAKE_WARM 568 +#define _MAKE_FUNCTION 570 +#define _MAKE_HEAP_SAFE 571 +#define _MAKE_WARM 572 #define _MAP_ADD MAP_ADD -#define _MATCH_CLASS 569 +#define _MATCH_CLASS 573 #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 570 -#define _MAYBE_EXPAND_METHOD_KW 571 -#define _MONITOR_CALL 572 -#define _MONITOR_CALL_KW 573 -#define _MONITOR_JUMP_BACKWARD 574 -#define _MONITOR_RESUME 575 +#define _MAYBE_EXPAND_METHOD 574 +#define _MAYBE_EXPAND_METHOD_KW 575 +#define _MONITOR_CALL 576 +#define _MONITOR_CALL_KW 577 +#define _MONITOR_JUMP_BACKWARD 578 +#define _MONITOR_RESUME 579 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 576 -#define _POP_JUMP_IF_TRUE 577 +#define _POP_JUMP_IF_FALSE 580 +#define _POP_JUMP_IF_TRUE 581 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 578 -#define _POP_TOP_INT 579 -#define _POP_TOP_NOP 580 -#define _POP_TOP_OPARG 581 -#define _POP_TOP_UNICODE 582 +#define _POP_TOP_FLOAT 582 +#define _POP_TOP_INT 583 +#define _POP_TOP_NOP 584 +#define _POP_TOP_OPARG 585 +#define _POP_TOP_UNICODE 586 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 583 +#define _PUSH_FRAME 587 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 584 -#define _PUSH_TAGGED_ZERO 585 -#define _PY_FRAME_EX 586 -#define _PY_FRAME_GENERAL 587 -#define _PY_FRAME_KW 588 -#define _RECORD_3OS_GEN_FUNC 589 -#define _RECORD_4OS 590 -#define _RECORD_BOUND_METHOD 591 -#define _RECORD_CALLABLE 592 -#define _RECORD_CALLABLE_KW 593 -#define _RECORD_CODE 594 -#define _RECORD_NOS 595 -#define _RECORD_NOS_GEN_FUNC 596 -#define _RECORD_NOS_TYPE 597 -#define _RECORD_TOS 598 -#define _RECORD_TOS_TYPE 599 -#define _REPLACE_WITH_TRUE 600 -#define _RESUME_CHECK 601 +#define _PUSH_NULL_CONDITIONAL 588 +#define _PUSH_TAGGED_ZERO 589 +#define _PY_FRAME_EX 590 +#define _PY_FRAME_GENERAL 591 +#define _PY_FRAME_KW 592 +#define _RECORD_3OS_GEN_FUNC 593 +#define _RECORD_4OS 594 +#define _RECORD_BOUND_METHOD 595 +#define _RECORD_CALLABLE 596 +#define _RECORD_CALLABLE_KW 597 +#define _RECORD_CODE 598 +#define _RECORD_NOS 599 +#define _RECORD_NOS_GEN_FUNC 600 +#define _RECORD_NOS_TYPE 601 +#define _RECORD_TOS 602 +#define _RECORD_TOS_TYPE 603 +#define _REPLACE_WITH_TRUE 604 +#define _RESUME_CHECK 605 #define _RETURN_GENERATOR RETURN_GENERATOR -#define _RETURN_VALUE 602 -#define _RROT_3 603 -#define _SAVE_RETURN_OFFSET 604 -#define _SEND 605 -#define _SEND_GEN_FRAME 606 +#define _RETURN_VALUE 606 +#define _RROT_3 607 +#define _SAVE_RETURN_OFFSET 608 +#define _SEND_ASYNC_GEN 609 +#define _SEND_ASYNC_GEN_TIER_TWO 610 +#define _SEND_GEN_FRAME 611 +#define _SEND_VIRTUAL 612 +#define _SEND_VIRTUAL_TIER_TWO 613 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _SET_UPDATE 607 -#define _SPILL_OR_RELOAD 608 -#define _START_EXECUTOR 609 -#define _STORE_ATTR 610 -#define _STORE_ATTR_INSTANCE_VALUE 611 -#define _STORE_ATTR_SLOT 612 -#define _STORE_ATTR_WITH_HINT 613 +#define _SET_UPDATE 614 +#define _SPILL_OR_RELOAD 615 +#define _START_EXECUTOR 616 +#define _STORE_ATTR 617 +#define _STORE_ATTR_INSTANCE_VALUE 618 +#define _STORE_ATTR_SLOT 619 +#define _STORE_ATTR_WITH_HINT 620 #define _STORE_DEREF STORE_DEREF #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 614 -#define _STORE_SUBSCR 615 -#define _STORE_SUBSCR_DICT 616 -#define _STORE_SUBSCR_DICT_KNOWN_HASH 617 -#define _STORE_SUBSCR_LIST_INT 618 -#define _SWAP 619 -#define _SWAP_2 620 -#define _SWAP_3 621 -#define _SWAP_FAST 622 -#define _SWAP_FAST_0 623 -#define _SWAP_FAST_1 624 -#define _SWAP_FAST_2 625 -#define _SWAP_FAST_3 626 -#define _SWAP_FAST_4 627 -#define _SWAP_FAST_5 628 -#define _SWAP_FAST_6 629 -#define _SWAP_FAST_7 630 -#define _TIER2_RESUME_CHECK 631 -#define _TO_BOOL 632 +#define _STORE_SLICE 621 +#define _STORE_SUBSCR 622 +#define _STORE_SUBSCR_DICT 623 +#define _STORE_SUBSCR_DICT_KNOWN_HASH 624 +#define _STORE_SUBSCR_LIST_INT 625 +#define _SWAP 626 +#define _SWAP_2 627 +#define _SWAP_3 628 +#define _SWAP_FAST 629 +#define _SWAP_FAST_0 630 +#define _SWAP_FAST_1 631 +#define _SWAP_FAST_2 632 +#define _SWAP_FAST_3 633 +#define _SWAP_FAST_4 634 +#define _SWAP_FAST_5 635 +#define _SWAP_FAST_6 636 +#define _SWAP_FAST_7 637 +#define _TIER2_RESUME_CHECK 638 +#define _TO_BOOL 639 #define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT 633 -#define _TO_BOOL_LIST 634 +#define _TO_BOOL_INT 640 +#define _TO_BOOL_LIST 641 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 635 +#define _TO_BOOL_STR 642 #define _TRACE_RECORD TRACE_RECORD -#define _UNARY_INVERT 636 -#define _UNARY_NEGATIVE 637 -#define _UNARY_NEGATIVE_FLOAT_INPLACE 638 +#define _UNARY_INVERT 643 +#define _UNARY_NEGATIVE 644 +#define _UNARY_NEGATIVE_FLOAT_INPLACE 645 #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 639 -#define _UNPACK_SEQUENCE_LIST 640 -#define _UNPACK_SEQUENCE_TUPLE 641 -#define _UNPACK_SEQUENCE_TWO_TUPLE 642 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE 643 -#define _UNPACK_SEQUENCE_UNIQUE_TUPLE 644 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE 645 +#define _UNPACK_SEQUENCE 646 +#define _UNPACK_SEQUENCE_LIST 647 +#define _UNPACK_SEQUENCE_TUPLE 648 +#define _UNPACK_SEQUENCE_TWO_TUPLE 649 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE 650 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE 651 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE 652 #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define _YIELD_VALUE 646 -#define MAX_UOP_ID 646 -#define _ALLOCATE_OBJECT_r00 647 -#define _BINARY_OP_r23 648 -#define _BINARY_OP_ADD_FLOAT_r03 649 -#define _BINARY_OP_ADD_FLOAT_r13 650 -#define _BINARY_OP_ADD_FLOAT_r23 651 -#define _BINARY_OP_ADD_FLOAT_INPLACE_r03 652 -#define _BINARY_OP_ADD_FLOAT_INPLACE_r13 653 -#define _BINARY_OP_ADD_FLOAT_INPLACE_r23 654 -#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 655 -#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 656 -#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 657 -#define _BINARY_OP_ADD_INT_r03 658 -#define _BINARY_OP_ADD_INT_r13 659 -#define _BINARY_OP_ADD_INT_r23 660 -#define _BINARY_OP_ADD_INT_INPLACE_r03 661 -#define _BINARY_OP_ADD_INT_INPLACE_r13 662 -#define _BINARY_OP_ADD_INT_INPLACE_r23 663 -#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 664 -#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 665 -#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 666 -#define _BINARY_OP_ADD_UNICODE_r03 667 -#define _BINARY_OP_ADD_UNICODE_r13 668 -#define _BINARY_OP_ADD_UNICODE_r23 669 -#define _BINARY_OP_EXTEND_r23 670 -#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 671 -#define _BINARY_OP_MULTIPLY_FLOAT_r03 672 -#define _BINARY_OP_MULTIPLY_FLOAT_r13 673 -#define _BINARY_OP_MULTIPLY_FLOAT_r23 674 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 675 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 676 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 677 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 678 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 679 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 680 -#define _BINARY_OP_MULTIPLY_INT_r03 681 -#define _BINARY_OP_MULTIPLY_INT_r13 682 -#define _BINARY_OP_MULTIPLY_INT_r23 683 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_r03 684 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_r13 685 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_r23 686 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 687 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 688 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 689 -#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 690 -#define _BINARY_OP_SUBSCR_DICT_r23 691 -#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 692 -#define _BINARY_OP_SUBSCR_INIT_CALL_r01 693 -#define _BINARY_OP_SUBSCR_INIT_CALL_r11 694 -#define _BINARY_OP_SUBSCR_INIT_CALL_r21 695 -#define _BINARY_OP_SUBSCR_INIT_CALL_r31 696 -#define _BINARY_OP_SUBSCR_LIST_INT_r23 697 -#define _BINARY_OP_SUBSCR_LIST_SLICE_r23 698 -#define _BINARY_OP_SUBSCR_STR_INT_r23 699 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 700 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 701 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 702 -#define _BINARY_OP_SUBSCR_USTR_INT_r23 703 -#define _BINARY_OP_SUBTRACT_FLOAT_r03 704 -#define _BINARY_OP_SUBTRACT_FLOAT_r13 705 -#define _BINARY_OP_SUBTRACT_FLOAT_r23 706 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 707 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 708 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 709 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 710 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 711 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 712 -#define _BINARY_OP_SUBTRACT_INT_r03 713 -#define _BINARY_OP_SUBTRACT_INT_r13 714 -#define _BINARY_OP_SUBTRACT_INT_r23 715 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_r03 716 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_r13 717 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_r23 718 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 719 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 720 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 721 -#define _BINARY_OP_TRUEDIV_FLOAT_r23 722 -#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 723 -#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 724 -#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 725 -#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 726 -#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 727 -#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 728 -#define _BINARY_SLICE_r31 729 -#define _BUILD_INTERPOLATION_r01 730 -#define _BUILD_LIST_r01 731 -#define _BUILD_MAP_r01 732 -#define _BUILD_SET_r01 733 -#define _BUILD_SLICE_r01 734 -#define _BUILD_STRING_r01 735 -#define _BUILD_TEMPLATE_r21 736 -#define _BUILD_TUPLE_r01 737 -#define _CALL_BUILTIN_CLASS_r00 738 -#define _CALL_BUILTIN_FAST_r00 739 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 740 -#define _CALL_BUILTIN_O_r03 741 -#define _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 742 -#define _CALL_INTRINSIC_1_r12 743 -#define _CALL_INTRINSIC_2_r23 744 -#define _CALL_ISINSTANCE_r31 745 -#define _CALL_KW_NON_PY_r11 746 -#define _CALL_LEN_r33 747 -#define _CALL_LIST_APPEND_r03 748 -#define _CALL_LIST_APPEND_r13 749 -#define _CALL_LIST_APPEND_r23 750 -#define _CALL_LIST_APPEND_r33 751 -#define _CALL_METHOD_DESCRIPTOR_FAST_r00 752 -#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 753 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 754 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 755 -#define _CALL_METHOD_DESCRIPTOR_NOARGS_r03 756 -#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 757 -#define _CALL_METHOD_DESCRIPTOR_O_r03 758 -#define _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 759 -#define _CALL_NON_PY_GENERAL_r01 760 -#define _CALL_STR_1_r32 761 -#define _CALL_TUPLE_1_r32 762 -#define _CALL_TYPE_1_r02 763 -#define _CALL_TYPE_1_r12 764 -#define _CALL_TYPE_1_r22 765 -#define _CALL_TYPE_1_r32 766 -#define _CHECK_ATTR_CLASS_r01 767 -#define _CHECK_ATTR_CLASS_r11 768 -#define _CHECK_ATTR_CLASS_r22 769 -#define _CHECK_ATTR_CLASS_r33 770 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 771 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 772 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 773 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 774 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 775 -#define _CHECK_EG_MATCH_r22 776 -#define _CHECK_EXC_MATCH_r22 777 -#define _CHECK_FUNCTION_EXACT_ARGS_r00 778 -#define _CHECK_FUNCTION_VERSION_r00 779 -#define _CHECK_FUNCTION_VERSION_INLINE_r00 780 -#define _CHECK_FUNCTION_VERSION_INLINE_r11 781 -#define _CHECK_FUNCTION_VERSION_INLINE_r22 782 -#define _CHECK_FUNCTION_VERSION_INLINE_r33 783 -#define _CHECK_FUNCTION_VERSION_KW_r11 784 -#define _CHECK_IS_NOT_PY_CALLABLE_r00 785 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r03 786 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r13 787 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r23 788 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r33 789 -#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 790 -#define _CHECK_IS_PY_CALLABLE_EX_r03 791 -#define _CHECK_IS_PY_CALLABLE_EX_r13 792 -#define _CHECK_IS_PY_CALLABLE_EX_r23 793 -#define _CHECK_IS_PY_CALLABLE_EX_r33 794 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 795 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 796 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 797 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 798 -#define _CHECK_METHOD_VERSION_r00 799 -#define _CHECK_METHOD_VERSION_KW_r11 800 -#define _CHECK_OBJECT_r00 801 -#define _CHECK_PEP_523_r00 802 -#define _CHECK_PEP_523_r11 803 -#define _CHECK_PEP_523_r22 804 -#define _CHECK_PEP_523_r33 805 -#define _CHECK_PERIODIC_r00 806 -#define _CHECK_PERIODIC_AT_END_r00 807 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 808 -#define _CHECK_RECURSION_LIMIT_r00 809 -#define _CHECK_RECURSION_LIMIT_r11 810 -#define _CHECK_RECURSION_LIMIT_r22 811 -#define _CHECK_RECURSION_LIMIT_r33 812 -#define _CHECK_RECURSION_REMAINING_r00 813 -#define _CHECK_RECURSION_REMAINING_r11 814 -#define _CHECK_RECURSION_REMAINING_r22 815 -#define _CHECK_RECURSION_REMAINING_r33 816 -#define _CHECK_STACK_SPACE_r00 817 -#define _CHECK_STACK_SPACE_OPERAND_r00 818 -#define _CHECK_STACK_SPACE_OPERAND_r11 819 -#define _CHECK_STACK_SPACE_OPERAND_r22 820 -#define _CHECK_STACK_SPACE_OPERAND_r33 821 -#define _CHECK_VALIDITY_r00 822 -#define _CHECK_VALIDITY_r11 823 -#define _CHECK_VALIDITY_r22 824 -#define _CHECK_VALIDITY_r33 825 -#define _COLD_DYNAMIC_EXIT_r00 826 -#define _COLD_EXIT_r00 827 -#define _COMPARE_OP_r21 828 -#define _COMPARE_OP_FLOAT_r03 829 -#define _COMPARE_OP_FLOAT_r13 830 -#define _COMPARE_OP_FLOAT_r23 831 -#define _COMPARE_OP_INT_r23 832 -#define _COMPARE_OP_STR_r23 833 -#define _CONTAINS_OP_r23 834 -#define _CONTAINS_OP_DICT_r23 835 -#define _CONTAINS_OP_SET_r23 836 -#define _CONVERT_VALUE_r11 837 -#define _COPY_r01 838 -#define _COPY_1_r02 839 -#define _COPY_1_r12 840 -#define _COPY_1_r23 841 -#define _COPY_2_r03 842 -#define _COPY_2_r13 843 -#define _COPY_2_r23 844 -#define _COPY_3_r03 845 -#define _COPY_3_r13 846 -#define _COPY_3_r23 847 -#define _COPY_3_r33 848 -#define _COPY_FREE_VARS_r00 849 -#define _COPY_FREE_VARS_r11 850 -#define _COPY_FREE_VARS_r22 851 -#define _COPY_FREE_VARS_r33 852 -#define _CREATE_INIT_FRAME_r01 853 -#define _DELETE_ATTR_r10 854 -#define _DELETE_DEREF_r00 855 -#define _DELETE_FAST_r00 856 -#define _DELETE_GLOBAL_r00 857 -#define _DELETE_NAME_r00 858 -#define _DELETE_SUBSCR_r20 859 -#define _DEOPT_r00 860 -#define _DEOPT_r10 861 -#define _DEOPT_r20 862 -#define _DEOPT_r30 863 -#define _DICT_MERGE_r11 864 -#define _DICT_UPDATE_r11 865 -#define _DO_CALL_r01 866 -#define _DO_CALL_FUNCTION_EX_r31 867 -#define _DO_CALL_KW_r11 868 -#define _DYNAMIC_EXIT_r00 869 -#define _DYNAMIC_EXIT_r10 870 -#define _DYNAMIC_EXIT_r20 871 -#define _DYNAMIC_EXIT_r30 872 -#define _END_FOR_r10 873 -#define _END_SEND_r31 874 -#define _ERROR_POP_N_r00 875 -#define _EXIT_INIT_CHECK_r10 876 -#define _EXIT_TRACE_r00 877 -#define _EXIT_TRACE_r10 878 -#define _EXIT_TRACE_r20 879 -#define _EXIT_TRACE_r30 880 -#define _EXPAND_METHOD_r00 881 -#define _EXPAND_METHOD_KW_r11 882 -#define _FATAL_ERROR_r00 883 -#define _FATAL_ERROR_r11 884 -#define _FATAL_ERROR_r22 885 -#define _FATAL_ERROR_r33 886 -#define _FORMAT_SIMPLE_r11 887 -#define _FORMAT_WITH_SPEC_r21 888 -#define _FOR_ITER_r23 889 -#define _FOR_ITER_GEN_FRAME_r03 890 -#define _FOR_ITER_GEN_FRAME_r13 891 -#define _FOR_ITER_GEN_FRAME_r23 892 -#define _FOR_ITER_TIER_TWO_r23 893 -#define _FOR_ITER_VIRTUAL_r23 894 -#define _FOR_ITER_VIRTUAL_TIER_TWO_r23 895 -#define _GET_AITER_r11 896 -#define _GET_ANEXT_r12 897 -#define _GET_AWAITABLE_r11 898 -#define _GET_ITER_r12 899 -#define _GET_ITER_TRAD_r12 900 -#define _GET_LEN_r12 901 -#define _GUARD_BINARY_OP_EXTEND_r22 902 -#define _GUARD_BINARY_OP_EXTEND_LHS_r02 903 -#define _GUARD_BINARY_OP_EXTEND_LHS_r12 904 -#define _GUARD_BINARY_OP_EXTEND_LHS_r22 905 -#define _GUARD_BINARY_OP_EXTEND_LHS_r33 906 -#define _GUARD_BINARY_OP_EXTEND_RHS_r02 907 -#define _GUARD_BINARY_OP_EXTEND_RHS_r12 908 -#define _GUARD_BINARY_OP_EXTEND_RHS_r22 909 -#define _GUARD_BINARY_OP_EXTEND_RHS_r33 910 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 911 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 912 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 913 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 914 -#define _GUARD_BIT_IS_SET_POP_r00 915 -#define _GUARD_BIT_IS_SET_POP_r10 916 -#define _GUARD_BIT_IS_SET_POP_r21 917 -#define _GUARD_BIT_IS_SET_POP_r32 918 -#define _GUARD_BIT_IS_SET_POP_4_r00 919 -#define _GUARD_BIT_IS_SET_POP_4_r10 920 -#define _GUARD_BIT_IS_SET_POP_4_r21 921 -#define _GUARD_BIT_IS_SET_POP_4_r32 922 -#define _GUARD_BIT_IS_SET_POP_5_r00 923 -#define _GUARD_BIT_IS_SET_POP_5_r10 924 -#define _GUARD_BIT_IS_SET_POP_5_r21 925 -#define _GUARD_BIT_IS_SET_POP_5_r32 926 -#define _GUARD_BIT_IS_SET_POP_6_r00 927 -#define _GUARD_BIT_IS_SET_POP_6_r10 928 -#define _GUARD_BIT_IS_SET_POP_6_r21 929 -#define _GUARD_BIT_IS_SET_POP_6_r32 930 -#define _GUARD_BIT_IS_SET_POP_7_r00 931 -#define _GUARD_BIT_IS_SET_POP_7_r10 932 -#define _GUARD_BIT_IS_SET_POP_7_r21 933 -#define _GUARD_BIT_IS_SET_POP_7_r32 934 -#define _GUARD_BIT_IS_UNSET_POP_r00 935 -#define _GUARD_BIT_IS_UNSET_POP_r10 936 -#define _GUARD_BIT_IS_UNSET_POP_r21 937 -#define _GUARD_BIT_IS_UNSET_POP_r32 938 -#define _GUARD_BIT_IS_UNSET_POP_4_r00 939 -#define _GUARD_BIT_IS_UNSET_POP_4_r10 940 -#define _GUARD_BIT_IS_UNSET_POP_4_r21 941 -#define _GUARD_BIT_IS_UNSET_POP_4_r32 942 -#define _GUARD_BIT_IS_UNSET_POP_5_r00 943 -#define _GUARD_BIT_IS_UNSET_POP_5_r10 944 -#define _GUARD_BIT_IS_UNSET_POP_5_r21 945 -#define _GUARD_BIT_IS_UNSET_POP_5_r32 946 -#define _GUARD_BIT_IS_UNSET_POP_6_r00 947 -#define _GUARD_BIT_IS_UNSET_POP_6_r10 948 -#define _GUARD_BIT_IS_UNSET_POP_6_r21 949 -#define _GUARD_BIT_IS_UNSET_POP_6_r32 950 -#define _GUARD_BIT_IS_UNSET_POP_7_r00 951 -#define _GUARD_BIT_IS_UNSET_POP_7_r10 952 -#define _GUARD_BIT_IS_UNSET_POP_7_r21 953 -#define _GUARD_BIT_IS_UNSET_POP_7_r32 954 -#define _GUARD_CALLABLE_BUILTIN_CLASS_r00 955 -#define _GUARD_CALLABLE_BUILTIN_FAST_r00 956 -#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 957 -#define _GUARD_CALLABLE_BUILTIN_O_r00 958 -#define _GUARD_CALLABLE_ISINSTANCE_r03 959 -#define _GUARD_CALLABLE_ISINSTANCE_r13 960 -#define _GUARD_CALLABLE_ISINSTANCE_r23 961 -#define _GUARD_CALLABLE_ISINSTANCE_r33 962 -#define _GUARD_CALLABLE_LEN_r03 963 -#define _GUARD_CALLABLE_LEN_r13 964 -#define _GUARD_CALLABLE_LEN_r23 965 -#define _GUARD_CALLABLE_LEN_r33 966 -#define _GUARD_CALLABLE_LIST_APPEND_r03 967 -#define _GUARD_CALLABLE_LIST_APPEND_r13 968 -#define _GUARD_CALLABLE_LIST_APPEND_r23 969 -#define _GUARD_CALLABLE_LIST_APPEND_r33 970 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 971 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 972 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 973 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 974 -#define _GUARD_CALLABLE_STR_1_r03 975 -#define _GUARD_CALLABLE_STR_1_r13 976 -#define _GUARD_CALLABLE_STR_1_r23 977 -#define _GUARD_CALLABLE_STR_1_r33 978 -#define _GUARD_CALLABLE_TUPLE_1_r03 979 -#define _GUARD_CALLABLE_TUPLE_1_r13 980 -#define _GUARD_CALLABLE_TUPLE_1_r23 981 -#define _GUARD_CALLABLE_TUPLE_1_r33 982 -#define _GUARD_CALLABLE_TYPE_1_r03 983 -#define _GUARD_CALLABLE_TYPE_1_r13 984 -#define _GUARD_CALLABLE_TYPE_1_r23 985 -#define _GUARD_CALLABLE_TYPE_1_r33 986 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 987 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 988 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 989 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 990 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r00 991 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r11 992 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r22 993 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r33 994 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r00 995 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r11 996 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r22 997 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r33 998 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r00 999 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r11 1000 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r22 1001 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r33 1002 -#define _GUARD_DORV_NO_DICT_r01 1003 -#define _GUARD_DORV_NO_DICT_r11 1004 -#define _GUARD_DORV_NO_DICT_r22 1005 -#define _GUARD_DORV_NO_DICT_r33 1006 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 1007 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 1008 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 1009 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 1010 -#define _GUARD_GLOBALS_VERSION_r00 1011 -#define _GUARD_GLOBALS_VERSION_r11 1012 -#define _GUARD_GLOBALS_VERSION_r22 1013 -#define _GUARD_GLOBALS_VERSION_r33 1014 -#define _GUARD_IP_RETURN_GENERATOR_r00 1015 -#define _GUARD_IP_RETURN_GENERATOR_r11 1016 -#define _GUARD_IP_RETURN_GENERATOR_r22 1017 -#define _GUARD_IP_RETURN_GENERATOR_r33 1018 -#define _GUARD_IP_RETURN_VALUE_r00 1019 -#define _GUARD_IP_RETURN_VALUE_r11 1020 -#define _GUARD_IP_RETURN_VALUE_r22 1021 -#define _GUARD_IP_RETURN_VALUE_r33 1022 -#define _GUARD_IP_YIELD_VALUE_r00 1023 -#define _GUARD_IP_YIELD_VALUE_r11 1024 -#define _GUARD_IP_YIELD_VALUE_r22 1025 -#define _GUARD_IP_YIELD_VALUE_r33 1026 -#define _GUARD_IP__PUSH_FRAME_r00 1027 -#define _GUARD_IP__PUSH_FRAME_r11 1028 -#define _GUARD_IP__PUSH_FRAME_r22 1029 -#define _GUARD_IP__PUSH_FRAME_r33 1030 -#define _GUARD_IS_FALSE_POP_r00 1031 -#define _GUARD_IS_FALSE_POP_r10 1032 -#define _GUARD_IS_FALSE_POP_r21 1033 -#define _GUARD_IS_FALSE_POP_r32 1034 -#define _GUARD_IS_NONE_POP_r00 1035 -#define _GUARD_IS_NONE_POP_r10 1036 -#define _GUARD_IS_NONE_POP_r21 1037 -#define _GUARD_IS_NONE_POP_r32 1038 -#define _GUARD_IS_NOT_NONE_POP_r10 1039 -#define _GUARD_IS_TRUE_POP_r00 1040 -#define _GUARD_IS_TRUE_POP_r10 1041 -#define _GUARD_IS_TRUE_POP_r21 1042 -#define _GUARD_IS_TRUE_POP_r32 1043 -#define _GUARD_ITERATOR_r01 1044 -#define _GUARD_ITERATOR_r11 1045 -#define _GUARD_ITERATOR_r22 1046 -#define _GUARD_ITERATOR_r33 1047 -#define _GUARD_ITER_VIRTUAL_r01 1048 -#define _GUARD_ITER_VIRTUAL_r11 1049 -#define _GUARD_ITER_VIRTUAL_r22 1050 -#define _GUARD_ITER_VIRTUAL_r33 1051 -#define _GUARD_KEYS_VERSION_r01 1052 -#define _GUARD_KEYS_VERSION_r11 1053 -#define _GUARD_KEYS_VERSION_r22 1054 -#define _GUARD_KEYS_VERSION_r33 1055 -#define _GUARD_LOAD_SUPER_ATTR_METHOD_r03 1056 -#define _GUARD_LOAD_SUPER_ATTR_METHOD_r13 1057 -#define _GUARD_LOAD_SUPER_ATTR_METHOD_r23 1058 -#define _GUARD_LOAD_SUPER_ATTR_METHOD_r33 1059 -#define _GUARD_NOS_COMPACT_ASCII_r02 1060 -#define _GUARD_NOS_COMPACT_ASCII_r12 1061 -#define _GUARD_NOS_COMPACT_ASCII_r22 1062 -#define _GUARD_NOS_COMPACT_ASCII_r33 1063 -#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03 1064 -#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13 1065 -#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23 1066 -#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33 1067 -#define _GUARD_NOS_DICT_SUBSCRIPT_r02 1068 -#define _GUARD_NOS_DICT_SUBSCRIPT_r12 1069 -#define _GUARD_NOS_DICT_SUBSCRIPT_r22 1070 -#define _GUARD_NOS_DICT_SUBSCRIPT_r33 1071 -#define _GUARD_NOS_FLOAT_r02 1072 -#define _GUARD_NOS_FLOAT_r12 1073 -#define _GUARD_NOS_FLOAT_r22 1074 -#define _GUARD_NOS_FLOAT_r33 1075 -#define _GUARD_NOS_INT_r02 1076 -#define _GUARD_NOS_INT_r12 1077 -#define _GUARD_NOS_INT_r22 1078 -#define _GUARD_NOS_INT_r33 1079 -#define _GUARD_NOS_ITER_VIRTUAL_r02 1080 -#define _GUARD_NOS_ITER_VIRTUAL_r12 1081 -#define _GUARD_NOS_ITER_VIRTUAL_r22 1082 -#define _GUARD_NOS_ITER_VIRTUAL_r33 1083 -#define _GUARD_NOS_LIST_r02 1084 -#define _GUARD_NOS_LIST_r12 1085 -#define _GUARD_NOS_LIST_r22 1086 -#define _GUARD_NOS_LIST_r33 1087 -#define _GUARD_NOS_NOT_NULL_r02 1088 -#define _GUARD_NOS_NOT_NULL_r12 1089 -#define _GUARD_NOS_NOT_NULL_r22 1090 -#define _GUARD_NOS_NOT_NULL_r33 1091 -#define _GUARD_NOS_NULL_r02 1092 -#define _GUARD_NOS_NULL_r12 1093 -#define _GUARD_NOS_NULL_r22 1094 -#define _GUARD_NOS_NULL_r33 1095 -#define _GUARD_NOS_OVERFLOWED_r02 1096 -#define _GUARD_NOS_OVERFLOWED_r12 1097 -#define _GUARD_NOS_OVERFLOWED_r22 1098 -#define _GUARD_NOS_OVERFLOWED_r33 1099 -#define _GUARD_NOS_TUPLE_r02 1100 -#define _GUARD_NOS_TUPLE_r12 1101 -#define _GUARD_NOS_TUPLE_r22 1102 -#define _GUARD_NOS_TUPLE_r33 1103 -#define _GUARD_NOS_TYPE_VERSION_r02 1104 -#define _GUARD_NOS_TYPE_VERSION_r12 1105 -#define _GUARD_NOS_TYPE_VERSION_r22 1106 -#define _GUARD_NOS_TYPE_VERSION_r33 1107 -#define _GUARD_NOS_UNICODE_r02 1108 -#define _GUARD_NOS_UNICODE_r12 1109 -#define _GUARD_NOS_UNICODE_r22 1110 -#define _GUARD_NOS_UNICODE_r33 1111 -#define _GUARD_NOT_EXHAUSTED_LIST_r02 1112 -#define _GUARD_NOT_EXHAUSTED_LIST_r12 1113 -#define _GUARD_NOT_EXHAUSTED_LIST_r22 1114 -#define _GUARD_NOT_EXHAUSTED_LIST_r33 1115 -#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1116 -#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1117 -#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1118 -#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1119 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1120 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1121 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1122 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1123 -#define _GUARD_THIRD_NULL_r03 1124 -#define _GUARD_THIRD_NULL_r13 1125 -#define _GUARD_THIRD_NULL_r23 1126 -#define _GUARD_THIRD_NULL_r33 1127 -#define _GUARD_TOS_ANY_DICT_r01 1128 -#define _GUARD_TOS_ANY_DICT_r11 1129 -#define _GUARD_TOS_ANY_DICT_r22 1130 -#define _GUARD_TOS_ANY_DICT_r33 1131 -#define _GUARD_TOS_ANY_SET_r01 1132 -#define _GUARD_TOS_ANY_SET_r11 1133 -#define _GUARD_TOS_ANY_SET_r22 1134 -#define _GUARD_TOS_ANY_SET_r33 1135 -#define _GUARD_TOS_DICT_r01 1136 -#define _GUARD_TOS_DICT_r11 1137 -#define _GUARD_TOS_DICT_r22 1138 -#define _GUARD_TOS_DICT_r33 1139 -#define _GUARD_TOS_FLOAT_r01 1140 -#define _GUARD_TOS_FLOAT_r11 1141 -#define _GUARD_TOS_FLOAT_r22 1142 -#define _GUARD_TOS_FLOAT_r33 1143 -#define _GUARD_TOS_FROZENDICT_r01 1144 -#define _GUARD_TOS_FROZENDICT_r11 1145 -#define _GUARD_TOS_FROZENDICT_r22 1146 -#define _GUARD_TOS_FROZENDICT_r33 1147 -#define _GUARD_TOS_FROZENSET_r01 1148 -#define _GUARD_TOS_FROZENSET_r11 1149 -#define _GUARD_TOS_FROZENSET_r22 1150 -#define _GUARD_TOS_FROZENSET_r33 1151 -#define _GUARD_TOS_INT_r01 1152 -#define _GUARD_TOS_INT_r11 1153 -#define _GUARD_TOS_INT_r22 1154 -#define _GUARD_TOS_INT_r33 1155 -#define _GUARD_TOS_LIST_r01 1156 -#define _GUARD_TOS_LIST_r11 1157 -#define _GUARD_TOS_LIST_r22 1158 -#define _GUARD_TOS_LIST_r33 1159 -#define _GUARD_TOS_OVERFLOWED_r01 1160 -#define _GUARD_TOS_OVERFLOWED_r11 1161 -#define _GUARD_TOS_OVERFLOWED_r22 1162 -#define _GUARD_TOS_OVERFLOWED_r33 1163 -#define _GUARD_TOS_SET_r01 1164 -#define _GUARD_TOS_SET_r11 1165 -#define _GUARD_TOS_SET_r22 1166 -#define _GUARD_TOS_SET_r33 1167 -#define _GUARD_TOS_SLICE_r01 1168 -#define _GUARD_TOS_SLICE_r11 1169 -#define _GUARD_TOS_SLICE_r22 1170 -#define _GUARD_TOS_SLICE_r33 1171 -#define _GUARD_TOS_TUPLE_r01 1172 -#define _GUARD_TOS_TUPLE_r11 1173 -#define _GUARD_TOS_TUPLE_r22 1174 -#define _GUARD_TOS_TUPLE_r33 1175 -#define _GUARD_TOS_UNICODE_r01 1176 -#define _GUARD_TOS_UNICODE_r11 1177 -#define _GUARD_TOS_UNICODE_r22 1178 -#define _GUARD_TOS_UNICODE_r33 1179 -#define _GUARD_TYPE_r01 1180 -#define _GUARD_TYPE_r11 1181 -#define _GUARD_TYPE_r22 1182 -#define _GUARD_TYPE_r33 1183 -#define _GUARD_TYPE_VERSION_r01 1184 -#define _GUARD_TYPE_VERSION_r11 1185 -#define _GUARD_TYPE_VERSION_r22 1186 -#define _GUARD_TYPE_VERSION_r33 1187 -#define _GUARD_TYPE_VERSION_LOCKED_r01 1188 -#define _GUARD_TYPE_VERSION_LOCKED_r11 1189 -#define _GUARD_TYPE_VERSION_LOCKED_r22 1190 -#define _GUARD_TYPE_VERSION_LOCKED_r33 1191 -#define _HANDLE_PENDING_AND_DEOPT_r00 1192 -#define _HANDLE_PENDING_AND_DEOPT_r10 1193 -#define _HANDLE_PENDING_AND_DEOPT_r20 1194 -#define _HANDLE_PENDING_AND_DEOPT_r30 1195 -#define _IMPORT_FROM_r12 1196 -#define _IMPORT_NAME_r21 1197 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1198 -#define _INIT_CALL_PY_EXACT_ARGS_r01 1199 -#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1200 -#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1201 -#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1202 -#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1203 -#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1204 -#define _INSERT_NULL_r10 1205 -#define _INSTRUMENTED_FOR_ITER_r23 1206 -#define _INSTRUMENTED_INSTRUCTION_r00 1207 -#define _INSTRUMENTED_JUMP_FORWARD_r00 1208 -#define _INSTRUMENTED_JUMP_FORWARD_r11 1209 -#define _INSTRUMENTED_JUMP_FORWARD_r22 1210 -#define _INSTRUMENTED_JUMP_FORWARD_r33 1211 -#define _INSTRUMENTED_LINE_r00 1212 -#define _INSTRUMENTED_NOT_TAKEN_r00 1213 -#define _INSTRUMENTED_NOT_TAKEN_r11 1214 -#define _INSTRUMENTED_NOT_TAKEN_r22 1215 -#define _INSTRUMENTED_NOT_TAKEN_r33 1216 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1217 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1218 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1219 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1220 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1221 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1222 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1223 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1224 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1225 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1226 -#define _IS_NONE_r11 1227 -#define _IS_OP_r03 1228 -#define _IS_OP_r13 1229 -#define _IS_OP_r23 1230 -#define _ITER_CHECK_LIST_r02 1231 -#define _ITER_CHECK_LIST_r12 1232 -#define _ITER_CHECK_LIST_r22 1233 -#define _ITER_CHECK_LIST_r33 1234 -#define _ITER_CHECK_RANGE_r02 1235 -#define _ITER_CHECK_RANGE_r12 1236 -#define _ITER_CHECK_RANGE_r22 1237 -#define _ITER_CHECK_RANGE_r33 1238 -#define _ITER_CHECK_TUPLE_r02 1239 -#define _ITER_CHECK_TUPLE_r12 1240 -#define _ITER_CHECK_TUPLE_r22 1241 -#define _ITER_CHECK_TUPLE_r33 1242 -#define _ITER_JUMP_LIST_r02 1243 -#define _ITER_JUMP_LIST_r12 1244 -#define _ITER_JUMP_LIST_r22 1245 -#define _ITER_JUMP_LIST_r33 1246 -#define _ITER_JUMP_RANGE_r02 1247 -#define _ITER_JUMP_RANGE_r12 1248 -#define _ITER_JUMP_RANGE_r22 1249 -#define _ITER_JUMP_RANGE_r33 1250 -#define _ITER_JUMP_TUPLE_r02 1251 -#define _ITER_JUMP_TUPLE_r12 1252 -#define _ITER_JUMP_TUPLE_r22 1253 -#define _ITER_JUMP_TUPLE_r33 1254 -#define _ITER_NEXT_LIST_r23 1255 -#define _ITER_NEXT_LIST_TIER_TWO_r23 1256 -#define _ITER_NEXT_RANGE_r03 1257 -#define _ITER_NEXT_RANGE_r13 1258 -#define _ITER_NEXT_RANGE_r23 1259 -#define _ITER_NEXT_TUPLE_r03 1260 -#define _ITER_NEXT_TUPLE_r13 1261 -#define _ITER_NEXT_TUPLE_r23 1262 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1263 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1264 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1265 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1266 -#define _JUMP_TO_TOP_r00 1267 -#define _LIST_APPEND_r10 1268 -#define _LIST_EXTEND_r11 1269 -#define _LOAD_ATTR_r10 1270 -#define _LOAD_ATTR_CLASS_r11 1271 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 1272 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 1273 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 1274 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 1275 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1276 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1277 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1278 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 1279 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 1280 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 1281 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1282 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1283 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1284 -#define _LOAD_ATTR_MODULE_r12 1285 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1286 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1287 -#define _LOAD_ATTR_PROPERTY_FRAME_r01 1288 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1289 -#define _LOAD_ATTR_PROPERTY_FRAME_r22 1290 -#define _LOAD_ATTR_PROPERTY_FRAME_r33 1291 -#define _LOAD_ATTR_SLOT_r02 1292 -#define _LOAD_ATTR_SLOT_r12 1293 -#define _LOAD_ATTR_SLOT_r23 1294 -#define _LOAD_ATTR_WITH_HINT_r12 1295 -#define _LOAD_BUILD_CLASS_r01 1296 -#define _LOAD_BYTECODE_r00 1297 -#define _LOAD_COMMON_CONSTANT_r01 1298 -#define _LOAD_COMMON_CONSTANT_r12 1299 -#define _LOAD_COMMON_CONSTANT_r23 1300 -#define _LOAD_CONST_r01 1301 -#define _LOAD_CONST_r12 1302 -#define _LOAD_CONST_r23 1303 -#define _LOAD_CONST_INLINE_r01 1304 -#define _LOAD_CONST_INLINE_r12 1305 -#define _LOAD_CONST_INLINE_r23 1306 -#define _LOAD_CONST_INLINE_BORROW_r01 1307 -#define _LOAD_CONST_INLINE_BORROW_r12 1308 -#define _LOAD_CONST_INLINE_BORROW_r23 1309 -#define _LOAD_DEREF_r01 1310 -#define _LOAD_FAST_r01 1311 -#define _LOAD_FAST_r12 1312 -#define _LOAD_FAST_r23 1313 -#define _LOAD_FAST_0_r01 1314 -#define _LOAD_FAST_0_r12 1315 -#define _LOAD_FAST_0_r23 1316 -#define _LOAD_FAST_1_r01 1317 -#define _LOAD_FAST_1_r12 1318 -#define _LOAD_FAST_1_r23 1319 -#define _LOAD_FAST_2_r01 1320 -#define _LOAD_FAST_2_r12 1321 -#define _LOAD_FAST_2_r23 1322 -#define _LOAD_FAST_3_r01 1323 -#define _LOAD_FAST_3_r12 1324 -#define _LOAD_FAST_3_r23 1325 -#define _LOAD_FAST_4_r01 1326 -#define _LOAD_FAST_4_r12 1327 -#define _LOAD_FAST_4_r23 1328 -#define _LOAD_FAST_5_r01 1329 -#define _LOAD_FAST_5_r12 1330 -#define _LOAD_FAST_5_r23 1331 -#define _LOAD_FAST_6_r01 1332 -#define _LOAD_FAST_6_r12 1333 -#define _LOAD_FAST_6_r23 1334 -#define _LOAD_FAST_7_r01 1335 -#define _LOAD_FAST_7_r12 1336 -#define _LOAD_FAST_7_r23 1337 -#define _LOAD_FAST_AND_CLEAR_r01 1338 -#define _LOAD_FAST_AND_CLEAR_r12 1339 -#define _LOAD_FAST_AND_CLEAR_r23 1340 -#define _LOAD_FAST_BORROW_r01 1341 -#define _LOAD_FAST_BORROW_r12 1342 -#define _LOAD_FAST_BORROW_r23 1343 -#define _LOAD_FAST_BORROW_0_r01 1344 -#define _LOAD_FAST_BORROW_0_r12 1345 -#define _LOAD_FAST_BORROW_0_r23 1346 -#define _LOAD_FAST_BORROW_1_r01 1347 -#define _LOAD_FAST_BORROW_1_r12 1348 -#define _LOAD_FAST_BORROW_1_r23 1349 -#define _LOAD_FAST_BORROW_2_r01 1350 -#define _LOAD_FAST_BORROW_2_r12 1351 -#define _LOAD_FAST_BORROW_2_r23 1352 -#define _LOAD_FAST_BORROW_3_r01 1353 -#define _LOAD_FAST_BORROW_3_r12 1354 -#define _LOAD_FAST_BORROW_3_r23 1355 -#define _LOAD_FAST_BORROW_4_r01 1356 -#define _LOAD_FAST_BORROW_4_r12 1357 -#define _LOAD_FAST_BORROW_4_r23 1358 -#define _LOAD_FAST_BORROW_5_r01 1359 -#define _LOAD_FAST_BORROW_5_r12 1360 -#define _LOAD_FAST_BORROW_5_r23 1361 -#define _LOAD_FAST_BORROW_6_r01 1362 -#define _LOAD_FAST_BORROW_6_r12 1363 -#define _LOAD_FAST_BORROW_6_r23 1364 -#define _LOAD_FAST_BORROW_7_r01 1365 -#define _LOAD_FAST_BORROW_7_r12 1366 -#define _LOAD_FAST_BORROW_7_r23 1367 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1368 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1369 -#define _LOAD_FAST_CHECK_r01 1370 -#define _LOAD_FAST_CHECK_r12 1371 -#define _LOAD_FAST_CHECK_r23 1372 -#define _LOAD_FAST_LOAD_FAST_r02 1373 -#define _LOAD_FAST_LOAD_FAST_r13 1374 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1375 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1376 -#define _LOAD_GLOBAL_r00 1377 -#define _LOAD_GLOBAL_BUILTINS_r01 1378 -#define _LOAD_GLOBAL_MODULE_r01 1379 -#define _LOAD_LOCALS_r01 1380 -#define _LOAD_LOCALS_r12 1381 -#define _LOAD_LOCALS_r23 1382 -#define _LOAD_NAME_r01 1383 -#define _LOAD_SMALL_INT_r01 1384 -#define _LOAD_SMALL_INT_r12 1385 -#define _LOAD_SMALL_INT_r23 1386 -#define _LOAD_SMALL_INT_0_r01 1387 -#define _LOAD_SMALL_INT_0_r12 1388 -#define _LOAD_SMALL_INT_0_r23 1389 -#define _LOAD_SMALL_INT_1_r01 1390 -#define _LOAD_SMALL_INT_1_r12 1391 -#define _LOAD_SMALL_INT_1_r23 1392 -#define _LOAD_SMALL_INT_2_r01 1393 -#define _LOAD_SMALL_INT_2_r12 1394 -#define _LOAD_SMALL_INT_2_r23 1395 -#define _LOAD_SMALL_INT_3_r01 1396 -#define _LOAD_SMALL_INT_3_r12 1397 -#define _LOAD_SMALL_INT_3_r23 1398 -#define _LOAD_SPECIAL_r00 1399 -#define _LOAD_SUPER_ATTR_ATTR_r31 1400 -#define _LOAD_SUPER_ATTR_METHOD_r32 1401 -#define _LOCK_OBJECT_r01 1402 -#define _LOCK_OBJECT_r11 1403 -#define _LOCK_OBJECT_r22 1404 -#define _LOCK_OBJECT_r33 1405 -#define _MAKE_CALLARGS_A_TUPLE_r33 1406 -#define _MAKE_CELL_r00 1407 -#define _MAKE_FUNCTION_r12 1408 -#define _MAKE_HEAP_SAFE_r01 1409 -#define _MAKE_HEAP_SAFE_r11 1410 -#define _MAKE_HEAP_SAFE_r22 1411 -#define _MAKE_HEAP_SAFE_r33 1412 -#define _MAKE_WARM_r00 1413 -#define _MAKE_WARM_r11 1414 -#define _MAKE_WARM_r22 1415 -#define _MAKE_WARM_r33 1416 -#define _MAP_ADD_r20 1417 -#define _MATCH_CLASS_r33 1418 -#define _MATCH_KEYS_r23 1419 -#define _MATCH_MAPPING_r02 1420 -#define _MATCH_MAPPING_r12 1421 -#define _MATCH_MAPPING_r23 1422 -#define _MATCH_SEQUENCE_r02 1423 -#define _MATCH_SEQUENCE_r12 1424 -#define _MATCH_SEQUENCE_r23 1425 -#define _MAYBE_EXPAND_METHOD_r00 1426 -#define _MAYBE_EXPAND_METHOD_KW_r11 1427 -#define _MONITOR_CALL_r00 1428 -#define _MONITOR_CALL_KW_r11 1429 -#define _MONITOR_JUMP_BACKWARD_r00 1430 -#define _MONITOR_JUMP_BACKWARD_r11 1431 -#define _MONITOR_JUMP_BACKWARD_r22 1432 -#define _MONITOR_JUMP_BACKWARD_r33 1433 -#define _MONITOR_RESUME_r00 1434 -#define _NOP_r00 1435 -#define _NOP_r11 1436 -#define _NOP_r22 1437 -#define _NOP_r33 1438 -#define _POP_EXCEPT_r10 1439 -#define _POP_ITER_r20 1440 -#define _POP_JUMP_IF_FALSE_r00 1441 -#define _POP_JUMP_IF_FALSE_r10 1442 -#define _POP_JUMP_IF_FALSE_r21 1443 -#define _POP_JUMP_IF_FALSE_r32 1444 -#define _POP_JUMP_IF_TRUE_r00 1445 -#define _POP_JUMP_IF_TRUE_r10 1446 -#define _POP_JUMP_IF_TRUE_r21 1447 -#define _POP_JUMP_IF_TRUE_r32 1448 -#define _POP_TOP_r10 1449 -#define _POP_TOP_FLOAT_r00 1450 -#define _POP_TOP_FLOAT_r10 1451 -#define _POP_TOP_FLOAT_r21 1452 -#define _POP_TOP_FLOAT_r32 1453 -#define _POP_TOP_INT_r00 1454 -#define _POP_TOP_INT_r10 1455 -#define _POP_TOP_INT_r21 1456 -#define _POP_TOP_INT_r32 1457 -#define _POP_TOP_NOP_r00 1458 -#define _POP_TOP_NOP_r10 1459 -#define _POP_TOP_NOP_r21 1460 -#define _POP_TOP_NOP_r32 1461 -#define _POP_TOP_OPARG_r00 1462 -#define _POP_TOP_UNICODE_r00 1463 -#define _POP_TOP_UNICODE_r10 1464 -#define _POP_TOP_UNICODE_r21 1465 -#define _POP_TOP_UNICODE_r32 1466 -#define _PUSH_EXC_INFO_r02 1467 -#define _PUSH_EXC_INFO_r12 1468 -#define _PUSH_EXC_INFO_r23 1469 -#define _PUSH_FRAME_r10 1470 -#define _PUSH_NULL_r01 1471 -#define _PUSH_NULL_r12 1472 -#define _PUSH_NULL_r23 1473 -#define _PUSH_NULL_CONDITIONAL_r00 1474 -#define _PUSH_TAGGED_ZERO_r01 1475 -#define _PUSH_TAGGED_ZERO_r12 1476 -#define _PUSH_TAGGED_ZERO_r23 1477 -#define _PY_FRAME_EX_r31 1478 -#define _PY_FRAME_GENERAL_r01 1479 -#define _PY_FRAME_KW_r11 1480 -#define _REPLACE_WITH_TRUE_r02 1481 -#define _REPLACE_WITH_TRUE_r12 1482 -#define _REPLACE_WITH_TRUE_r23 1483 -#define _RESUME_CHECK_r00 1484 -#define _RESUME_CHECK_r11 1485 -#define _RESUME_CHECK_r22 1486 -#define _RESUME_CHECK_r33 1487 -#define _RETURN_GENERATOR_r01 1488 -#define _RETURN_VALUE_r11 1489 -#define _RROT_3_r03 1490 -#define _RROT_3_r13 1491 -#define _RROT_3_r23 1492 -#define _RROT_3_r33 1493 -#define _SAVE_RETURN_OFFSET_r00 1494 -#define _SAVE_RETURN_OFFSET_r11 1495 -#define _SAVE_RETURN_OFFSET_r22 1496 -#define _SAVE_RETURN_OFFSET_r33 1497 -#define _SEND_r33 1498 -#define _SEND_GEN_FRAME_r33 1499 -#define _SETUP_ANNOTATIONS_r00 1500 -#define _SET_ADD_r10 1501 -#define _SET_FUNCTION_ATTRIBUTE_r01 1502 -#define _SET_FUNCTION_ATTRIBUTE_r11 1503 -#define _SET_FUNCTION_ATTRIBUTE_r21 1504 -#define _SET_FUNCTION_ATTRIBUTE_r32 1505 -#define _SET_IP_r00 1506 -#define _SET_IP_r11 1507 -#define _SET_IP_r22 1508 -#define _SET_IP_r33 1509 -#define _SET_UPDATE_r11 1510 -#define _SPILL_OR_RELOAD_r01 1511 -#define _SPILL_OR_RELOAD_r02 1512 -#define _SPILL_OR_RELOAD_r03 1513 -#define _SPILL_OR_RELOAD_r10 1514 -#define _SPILL_OR_RELOAD_r12 1515 -#define _SPILL_OR_RELOAD_r13 1516 -#define _SPILL_OR_RELOAD_r20 1517 -#define _SPILL_OR_RELOAD_r21 1518 -#define _SPILL_OR_RELOAD_r23 1519 -#define _SPILL_OR_RELOAD_r30 1520 -#define _SPILL_OR_RELOAD_r31 1521 -#define _SPILL_OR_RELOAD_r32 1522 -#define _START_EXECUTOR_r00 1523 -#define _STORE_ATTR_r20 1524 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1525 -#define _STORE_ATTR_SLOT_r21 1526 -#define _STORE_ATTR_WITH_HINT_r21 1527 -#define _STORE_DEREF_r10 1528 -#define _STORE_FAST_LOAD_FAST_r11 1529 -#define _STORE_FAST_STORE_FAST_r20 1530 -#define _STORE_GLOBAL_r10 1531 -#define _STORE_NAME_r10 1532 -#define _STORE_SLICE_r30 1533 -#define _STORE_SUBSCR_r30 1534 -#define _STORE_SUBSCR_DICT_r31 1535 -#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1536 -#define _STORE_SUBSCR_LIST_INT_r32 1537 -#define _SWAP_r11 1538 -#define _SWAP_2_r02 1539 -#define _SWAP_2_r12 1540 -#define _SWAP_2_r22 1541 -#define _SWAP_2_r33 1542 -#define _SWAP_3_r03 1543 -#define _SWAP_3_r13 1544 -#define _SWAP_3_r23 1545 -#define _SWAP_3_r33 1546 -#define _SWAP_FAST_r01 1547 -#define _SWAP_FAST_r11 1548 -#define _SWAP_FAST_r22 1549 -#define _SWAP_FAST_r33 1550 -#define _SWAP_FAST_0_r01 1551 -#define _SWAP_FAST_0_r11 1552 -#define _SWAP_FAST_0_r22 1553 -#define _SWAP_FAST_0_r33 1554 -#define _SWAP_FAST_1_r01 1555 -#define _SWAP_FAST_1_r11 1556 -#define _SWAP_FAST_1_r22 1557 -#define _SWAP_FAST_1_r33 1558 -#define _SWAP_FAST_2_r01 1559 -#define _SWAP_FAST_2_r11 1560 -#define _SWAP_FAST_2_r22 1561 -#define _SWAP_FAST_2_r33 1562 -#define _SWAP_FAST_3_r01 1563 -#define _SWAP_FAST_3_r11 1564 -#define _SWAP_FAST_3_r22 1565 -#define _SWAP_FAST_3_r33 1566 -#define _SWAP_FAST_4_r01 1567 -#define _SWAP_FAST_4_r11 1568 -#define _SWAP_FAST_4_r22 1569 -#define _SWAP_FAST_4_r33 1570 -#define _SWAP_FAST_5_r01 1571 -#define _SWAP_FAST_5_r11 1572 -#define _SWAP_FAST_5_r22 1573 -#define _SWAP_FAST_5_r33 1574 -#define _SWAP_FAST_6_r01 1575 -#define _SWAP_FAST_6_r11 1576 -#define _SWAP_FAST_6_r22 1577 -#define _SWAP_FAST_6_r33 1578 -#define _SWAP_FAST_7_r01 1579 -#define _SWAP_FAST_7_r11 1580 -#define _SWAP_FAST_7_r22 1581 -#define _SWAP_FAST_7_r33 1582 -#define _TIER2_RESUME_CHECK_r00 1583 -#define _TIER2_RESUME_CHECK_r11 1584 -#define _TIER2_RESUME_CHECK_r22 1585 -#define _TIER2_RESUME_CHECK_r33 1586 -#define _TO_BOOL_r11 1587 -#define _TO_BOOL_BOOL_r01 1588 -#define _TO_BOOL_BOOL_r11 1589 -#define _TO_BOOL_BOOL_r22 1590 -#define _TO_BOOL_BOOL_r33 1591 -#define _TO_BOOL_INT_r02 1592 -#define _TO_BOOL_INT_r12 1593 -#define _TO_BOOL_INT_r23 1594 -#define _TO_BOOL_LIST_r02 1595 -#define _TO_BOOL_LIST_r12 1596 -#define _TO_BOOL_LIST_r23 1597 -#define _TO_BOOL_NONE_r01 1598 -#define _TO_BOOL_NONE_r11 1599 -#define _TO_BOOL_NONE_r22 1600 -#define _TO_BOOL_NONE_r33 1601 -#define _TO_BOOL_STR_r02 1602 -#define _TO_BOOL_STR_r12 1603 -#define _TO_BOOL_STR_r23 1604 -#define _TRACE_RECORD_r00 1605 -#define _UNARY_INVERT_r12 1606 -#define _UNARY_NEGATIVE_r12 1607 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1608 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1609 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1610 -#define _UNARY_NOT_r01 1611 -#define _UNARY_NOT_r11 1612 -#define _UNARY_NOT_r22 1613 -#define _UNARY_NOT_r33 1614 -#define _UNPACK_EX_r10 1615 -#define _UNPACK_SEQUENCE_r10 1616 -#define _UNPACK_SEQUENCE_LIST_r10 1617 -#define _UNPACK_SEQUENCE_TUPLE_r10 1618 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1619 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1620 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1621 -#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1622 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1623 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1624 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1625 -#define _WITH_EXCEPT_START_r33 1626 -#define _YIELD_VALUE_r11 1627 -#define MAX_UOP_REGS_ID 1627 +#define _YIELD_VALUE 653 +#define MAX_UOP_ID 653 +#define _ALLOCATE_OBJECT_r00 654 +#define _BINARY_OP_r23 655 +#define _BINARY_OP_ADD_FLOAT_r03 656 +#define _BINARY_OP_ADD_FLOAT_r13 657 +#define _BINARY_OP_ADD_FLOAT_r23 658 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r03 659 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r13 660 +#define _BINARY_OP_ADD_FLOAT_INPLACE_r23 661 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 662 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 663 +#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 664 +#define _BINARY_OP_ADD_INT_r03 665 +#define _BINARY_OP_ADD_INT_r13 666 +#define _BINARY_OP_ADD_INT_r23 667 +#define _BINARY_OP_ADD_INT_INPLACE_r03 668 +#define _BINARY_OP_ADD_INT_INPLACE_r13 669 +#define _BINARY_OP_ADD_INT_INPLACE_r23 670 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 671 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 672 +#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 673 +#define _BINARY_OP_ADD_UNICODE_r03 674 +#define _BINARY_OP_ADD_UNICODE_r13 675 +#define _BINARY_OP_ADD_UNICODE_r23 676 +#define _BINARY_OP_EXTEND_r23 677 +#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 678 +#define _BINARY_OP_MULTIPLY_FLOAT_r03 679 +#define _BINARY_OP_MULTIPLY_FLOAT_r13 680 +#define _BINARY_OP_MULTIPLY_FLOAT_r23 681 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 682 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 683 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 684 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 685 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 686 +#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 687 +#define _BINARY_OP_MULTIPLY_INT_r03 688 +#define _BINARY_OP_MULTIPLY_INT_r13 689 +#define _BINARY_OP_MULTIPLY_INT_r23 690 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r03 691 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r13 692 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_r23 693 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 694 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 695 +#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 696 +#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 697 +#define _BINARY_OP_SUBSCR_DICT_r23 698 +#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 699 +#define _BINARY_OP_SUBSCR_INIT_CALL_r01 700 +#define _BINARY_OP_SUBSCR_INIT_CALL_r11 701 +#define _BINARY_OP_SUBSCR_INIT_CALL_r21 702 +#define _BINARY_OP_SUBSCR_INIT_CALL_r31 703 +#define _BINARY_OP_SUBSCR_LIST_INT_r23 704 +#define _BINARY_OP_SUBSCR_LIST_SLICE_r23 705 +#define _BINARY_OP_SUBSCR_STR_INT_r23 706 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 707 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 708 +#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 709 +#define _BINARY_OP_SUBSCR_USTR_INT_r23 710 +#define _BINARY_OP_SUBTRACT_FLOAT_r03 711 +#define _BINARY_OP_SUBTRACT_FLOAT_r13 712 +#define _BINARY_OP_SUBTRACT_FLOAT_r23 713 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 714 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 715 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 716 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 717 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 718 +#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 719 +#define _BINARY_OP_SUBTRACT_INT_r03 720 +#define _BINARY_OP_SUBTRACT_INT_r13 721 +#define _BINARY_OP_SUBTRACT_INT_r23 722 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r03 723 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r13 724 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_r23 725 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 726 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 727 +#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 728 +#define _BINARY_OP_TRUEDIV_FLOAT_r23 729 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 730 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 731 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 732 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 733 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 734 +#define _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 735 +#define _BINARY_SLICE_r31 736 +#define _BUILD_INTERPOLATION_r01 737 +#define _BUILD_LIST_r01 738 +#define _BUILD_MAP_r01 739 +#define _BUILD_SET_r01 740 +#define _BUILD_SLICE_r01 741 +#define _BUILD_STRING_r01 742 +#define _BUILD_TEMPLATE_r21 743 +#define _BUILD_TUPLE_r01 744 +#define _CALL_BUILTIN_CLASS_r00 745 +#define _CALL_BUILTIN_FAST_r00 746 +#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 747 +#define _CALL_BUILTIN_O_r03 748 +#define _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 749 +#define _CALL_INTRINSIC_1_r12 750 +#define _CALL_INTRINSIC_2_r23 751 +#define _CALL_ISINSTANCE_r31 752 +#define _CALL_KW_NON_PY_r11 753 +#define _CALL_LEN_r33 754 +#define _CALL_LIST_APPEND_r03 755 +#define _CALL_LIST_APPEND_r13 756 +#define _CALL_LIST_APPEND_r23 757 +#define _CALL_LIST_APPEND_r33 758 +#define _CALL_METHOD_DESCRIPTOR_FAST_r00 759 +#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 760 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 761 +#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 762 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_r03 763 +#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 764 +#define _CALL_METHOD_DESCRIPTOR_O_r03 765 +#define _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 766 +#define _CALL_NON_PY_GENERAL_r01 767 +#define _CALL_STR_1_r32 768 +#define _CALL_TUPLE_1_r32 769 +#define _CALL_TYPE_1_r02 770 +#define _CALL_TYPE_1_r12 771 +#define _CALL_TYPE_1_r22 772 +#define _CALL_TYPE_1_r32 773 +#define _CHECK_ATTR_CLASS_r01 774 +#define _CHECK_ATTR_CLASS_r11 775 +#define _CHECK_ATTR_CLASS_r22 776 +#define _CHECK_ATTR_CLASS_r33 777 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 778 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 779 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 780 +#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 781 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 782 +#define _CHECK_EG_MATCH_r22 783 +#define _CHECK_EXC_MATCH_r22 784 +#define _CHECK_FUNCTION_EXACT_ARGS_r00 785 +#define _CHECK_FUNCTION_VERSION_r00 786 +#define _CHECK_FUNCTION_VERSION_INLINE_r00 787 +#define _CHECK_FUNCTION_VERSION_INLINE_r11 788 +#define _CHECK_FUNCTION_VERSION_INLINE_r22 789 +#define _CHECK_FUNCTION_VERSION_INLINE_r33 790 +#define _CHECK_FUNCTION_VERSION_KW_r11 791 +#define _CHECK_IS_NOT_PY_CALLABLE_r00 792 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r03 793 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r13 794 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r23 795 +#define _CHECK_IS_NOT_PY_CALLABLE_EX_r33 796 +#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 797 +#define _CHECK_IS_PY_CALLABLE_EX_r03 798 +#define _CHECK_IS_PY_CALLABLE_EX_r13 799 +#define _CHECK_IS_PY_CALLABLE_EX_r23 800 +#define _CHECK_IS_PY_CALLABLE_EX_r33 801 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 802 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 803 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 804 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 805 +#define _CHECK_METHOD_VERSION_r00 806 +#define _CHECK_METHOD_VERSION_KW_r11 807 +#define _CHECK_OBJECT_r00 808 +#define _CHECK_PEP_523_r00 809 +#define _CHECK_PEP_523_r11 810 +#define _CHECK_PEP_523_r22 811 +#define _CHECK_PEP_523_r33 812 +#define _CHECK_PERIODIC_r00 813 +#define _CHECK_PERIODIC_AT_END_r00 814 +#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 815 +#define _CHECK_RECURSION_LIMIT_r00 816 +#define _CHECK_RECURSION_LIMIT_r11 817 +#define _CHECK_RECURSION_LIMIT_r22 818 +#define _CHECK_RECURSION_LIMIT_r33 819 +#define _CHECK_RECURSION_REMAINING_r00 820 +#define _CHECK_RECURSION_REMAINING_r11 821 +#define _CHECK_RECURSION_REMAINING_r22 822 +#define _CHECK_RECURSION_REMAINING_r33 823 +#define _CHECK_STACK_SPACE_r00 824 +#define _CHECK_STACK_SPACE_OPERAND_r00 825 +#define _CHECK_STACK_SPACE_OPERAND_r11 826 +#define _CHECK_STACK_SPACE_OPERAND_r22 827 +#define _CHECK_STACK_SPACE_OPERAND_r33 828 +#define _CHECK_VALIDITY_r00 829 +#define _CHECK_VALIDITY_r11 830 +#define _CHECK_VALIDITY_r22 831 +#define _CHECK_VALIDITY_r33 832 +#define _COLD_DYNAMIC_EXIT_r00 833 +#define _COLD_EXIT_r00 834 +#define _COMPARE_OP_r21 835 +#define _COMPARE_OP_FLOAT_r03 836 +#define _COMPARE_OP_FLOAT_r13 837 +#define _COMPARE_OP_FLOAT_r23 838 +#define _COMPARE_OP_INT_r23 839 +#define _COMPARE_OP_STR_r23 840 +#define _CONTAINS_OP_r23 841 +#define _CONTAINS_OP_DICT_r23 842 +#define _CONTAINS_OP_SET_r23 843 +#define _CONVERT_VALUE_r11 844 +#define _COPY_r01 845 +#define _COPY_1_r02 846 +#define _COPY_1_r12 847 +#define _COPY_1_r23 848 +#define _COPY_2_r03 849 +#define _COPY_2_r13 850 +#define _COPY_2_r23 851 +#define _COPY_3_r03 852 +#define _COPY_3_r13 853 +#define _COPY_3_r23 854 +#define _COPY_3_r33 855 +#define _COPY_FREE_VARS_r00 856 +#define _COPY_FREE_VARS_r11 857 +#define _COPY_FREE_VARS_r22 858 +#define _COPY_FREE_VARS_r33 859 +#define _CREATE_INIT_FRAME_r01 860 +#define _DELETE_ATTR_r10 861 +#define _DELETE_DEREF_r00 862 +#define _DELETE_FAST_r00 863 +#define _DELETE_GLOBAL_r00 864 +#define _DELETE_NAME_r00 865 +#define _DELETE_SUBSCR_r20 866 +#define _DEOPT_r00 867 +#define _DEOPT_r10 868 +#define _DEOPT_r20 869 +#define _DEOPT_r30 870 +#define _DICT_MERGE_r11 871 +#define _DICT_UPDATE_r11 872 +#define _DO_CALL_r01 873 +#define _DO_CALL_FUNCTION_EX_r31 874 +#define _DO_CALL_KW_r11 875 +#define _DYNAMIC_EXIT_r00 876 +#define _DYNAMIC_EXIT_r10 877 +#define _DYNAMIC_EXIT_r20 878 +#define _DYNAMIC_EXIT_r30 879 +#define _END_FOR_r10 880 +#define _END_SEND_r31 881 +#define _ERROR_POP_N_r00 882 +#define _EXIT_INIT_CHECK_r10 883 +#define _EXIT_TRACE_r00 884 +#define _EXIT_TRACE_r10 885 +#define _EXIT_TRACE_r20 886 +#define _EXIT_TRACE_r30 887 +#define _EXPAND_METHOD_r00 888 +#define _EXPAND_METHOD_KW_r11 889 +#define _FATAL_ERROR_r00 890 +#define _FATAL_ERROR_r11 891 +#define _FATAL_ERROR_r22 892 +#define _FATAL_ERROR_r33 893 +#define _FORMAT_SIMPLE_r11 894 +#define _FORMAT_WITH_SPEC_r21 895 +#define _FOR_ITER_r23 896 +#define _FOR_ITER_GEN_FRAME_r03 897 +#define _FOR_ITER_GEN_FRAME_r13 898 +#define _FOR_ITER_GEN_FRAME_r23 899 +#define _FOR_ITER_TIER_TWO_r23 900 +#define _FOR_ITER_VIRTUAL_r23 901 +#define _FOR_ITER_VIRTUAL_TIER_TWO_r23 902 +#define _GET_AITER_r11 903 +#define _GET_ANEXT_r12 904 +#define _GET_AWAITABLE_r11 905 +#define _GET_ITER_r12 906 +#define _GET_ITER_TRAD_r12 907 +#define _GET_LEN_r12 908 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r03 909 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r13 910 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r23 911 +#define _GUARD_3OS_ASYNC_GEN_ASEND_r33 912 +#define _GUARD_BINARY_OP_EXTEND_r22 913 +#define _GUARD_BINARY_OP_EXTEND_LHS_r02 914 +#define _GUARD_BINARY_OP_EXTEND_LHS_r12 915 +#define _GUARD_BINARY_OP_EXTEND_LHS_r22 916 +#define _GUARD_BINARY_OP_EXTEND_LHS_r33 917 +#define _GUARD_BINARY_OP_EXTEND_RHS_r02 918 +#define _GUARD_BINARY_OP_EXTEND_RHS_r12 919 +#define _GUARD_BINARY_OP_EXTEND_RHS_r22 920 +#define _GUARD_BINARY_OP_EXTEND_RHS_r33 921 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 922 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 923 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 924 +#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 925 +#define _GUARD_BIT_IS_SET_POP_r00 926 +#define _GUARD_BIT_IS_SET_POP_r10 927 +#define _GUARD_BIT_IS_SET_POP_r21 928 +#define _GUARD_BIT_IS_SET_POP_r32 929 +#define _GUARD_BIT_IS_SET_POP_4_r00 930 +#define _GUARD_BIT_IS_SET_POP_4_r10 931 +#define _GUARD_BIT_IS_SET_POP_4_r21 932 +#define _GUARD_BIT_IS_SET_POP_4_r32 933 +#define _GUARD_BIT_IS_SET_POP_5_r00 934 +#define _GUARD_BIT_IS_SET_POP_5_r10 935 +#define _GUARD_BIT_IS_SET_POP_5_r21 936 +#define _GUARD_BIT_IS_SET_POP_5_r32 937 +#define _GUARD_BIT_IS_SET_POP_6_r00 938 +#define _GUARD_BIT_IS_SET_POP_6_r10 939 +#define _GUARD_BIT_IS_SET_POP_6_r21 940 +#define _GUARD_BIT_IS_SET_POP_6_r32 941 +#define _GUARD_BIT_IS_SET_POP_7_r00 942 +#define _GUARD_BIT_IS_SET_POP_7_r10 943 +#define _GUARD_BIT_IS_SET_POP_7_r21 944 +#define _GUARD_BIT_IS_SET_POP_7_r32 945 +#define _GUARD_BIT_IS_UNSET_POP_r00 946 +#define _GUARD_BIT_IS_UNSET_POP_r10 947 +#define _GUARD_BIT_IS_UNSET_POP_r21 948 +#define _GUARD_BIT_IS_UNSET_POP_r32 949 +#define _GUARD_BIT_IS_UNSET_POP_4_r00 950 +#define _GUARD_BIT_IS_UNSET_POP_4_r10 951 +#define _GUARD_BIT_IS_UNSET_POP_4_r21 952 +#define _GUARD_BIT_IS_UNSET_POP_4_r32 953 +#define _GUARD_BIT_IS_UNSET_POP_5_r00 954 +#define _GUARD_BIT_IS_UNSET_POP_5_r10 955 +#define _GUARD_BIT_IS_UNSET_POP_5_r21 956 +#define _GUARD_BIT_IS_UNSET_POP_5_r32 957 +#define _GUARD_BIT_IS_UNSET_POP_6_r00 958 +#define _GUARD_BIT_IS_UNSET_POP_6_r10 959 +#define _GUARD_BIT_IS_UNSET_POP_6_r21 960 +#define _GUARD_BIT_IS_UNSET_POP_6_r32 961 +#define _GUARD_BIT_IS_UNSET_POP_7_r00 962 +#define _GUARD_BIT_IS_UNSET_POP_7_r10 963 +#define _GUARD_BIT_IS_UNSET_POP_7_r21 964 +#define _GUARD_BIT_IS_UNSET_POP_7_r32 965 +#define _GUARD_CALLABLE_BUILTIN_CLASS_r00 966 +#define _GUARD_CALLABLE_BUILTIN_FAST_r00 967 +#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 968 +#define _GUARD_CALLABLE_BUILTIN_O_r00 969 +#define _GUARD_CALLABLE_ISINSTANCE_r03 970 +#define _GUARD_CALLABLE_ISINSTANCE_r13 971 +#define _GUARD_CALLABLE_ISINSTANCE_r23 972 +#define _GUARD_CALLABLE_ISINSTANCE_r33 973 +#define _GUARD_CALLABLE_LEN_r03 974 +#define _GUARD_CALLABLE_LEN_r13 975 +#define _GUARD_CALLABLE_LEN_r23 976 +#define _GUARD_CALLABLE_LEN_r33 977 +#define _GUARD_CALLABLE_LIST_APPEND_r03 978 +#define _GUARD_CALLABLE_LIST_APPEND_r13 979 +#define _GUARD_CALLABLE_LIST_APPEND_r23 980 +#define _GUARD_CALLABLE_LIST_APPEND_r33 981 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 982 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 983 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 984 +#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 985 +#define _GUARD_CALLABLE_STR_1_r03 986 +#define _GUARD_CALLABLE_STR_1_r13 987 +#define _GUARD_CALLABLE_STR_1_r23 988 +#define _GUARD_CALLABLE_STR_1_r33 989 +#define _GUARD_CALLABLE_TUPLE_1_r03 990 +#define _GUARD_CALLABLE_TUPLE_1_r13 991 +#define _GUARD_CALLABLE_TUPLE_1_r23 992 +#define _GUARD_CALLABLE_TUPLE_1_r33 993 +#define _GUARD_CALLABLE_TYPE_1_r03 994 +#define _GUARD_CALLABLE_TYPE_1_r13 995 +#define _GUARD_CALLABLE_TYPE_1_r23 996 +#define _GUARD_CALLABLE_TYPE_1_r33 997 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 998 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 999 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 1000 +#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 1001 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r00 1002 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r11 1003 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r22 1004 +#define _GUARD_CODE_VERSION_RETURN_VALUE_r33 1005 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r00 1006 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r11 1007 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r22 1008 +#define _GUARD_CODE_VERSION_YIELD_VALUE_r33 1009 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r00 1010 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r11 1011 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r22 1012 +#define _GUARD_CODE_VERSION__PUSH_FRAME_r33 1013 +#define _GUARD_DORV_NO_DICT_r01 1014 +#define _GUARD_DORV_NO_DICT_r11 1015 +#define _GUARD_DORV_NO_DICT_r22 1016 +#define _GUARD_DORV_NO_DICT_r33 1017 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 1018 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 1019 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 1020 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 1021 +#define _GUARD_GLOBALS_VERSION_r00 1022 +#define _GUARD_GLOBALS_VERSION_r11 1023 +#define _GUARD_GLOBALS_VERSION_r22 1024 +#define _GUARD_GLOBALS_VERSION_r33 1025 +#define _GUARD_IP_RETURN_GENERATOR_r00 1026 +#define _GUARD_IP_RETURN_GENERATOR_r11 1027 +#define _GUARD_IP_RETURN_GENERATOR_r22 1028 +#define _GUARD_IP_RETURN_GENERATOR_r33 1029 +#define _GUARD_IP_RETURN_VALUE_r00 1030 +#define _GUARD_IP_RETURN_VALUE_r11 1031 +#define _GUARD_IP_RETURN_VALUE_r22 1032 +#define _GUARD_IP_RETURN_VALUE_r33 1033 +#define _GUARD_IP_YIELD_VALUE_r00 1034 +#define _GUARD_IP_YIELD_VALUE_r11 1035 +#define _GUARD_IP_YIELD_VALUE_r22 1036 +#define _GUARD_IP_YIELD_VALUE_r33 1037 +#define _GUARD_IP__PUSH_FRAME_r00 1038 +#define _GUARD_IP__PUSH_FRAME_r11 1039 +#define _GUARD_IP__PUSH_FRAME_r22 1040 +#define _GUARD_IP__PUSH_FRAME_r33 1041 +#define _GUARD_IS_FALSE_POP_r00 1042 +#define _GUARD_IS_FALSE_POP_r10 1043 +#define _GUARD_IS_FALSE_POP_r21 1044 +#define _GUARD_IS_FALSE_POP_r32 1045 +#define _GUARD_IS_NONE_POP_r00 1046 +#define _GUARD_IS_NONE_POP_r10 1047 +#define _GUARD_IS_NONE_POP_r21 1048 +#define _GUARD_IS_NONE_POP_r32 1049 +#define _GUARD_IS_NOT_NONE_POP_r10 1050 +#define _GUARD_IS_TRUE_POP_r00 1051 +#define _GUARD_IS_TRUE_POP_r10 1052 +#define _GUARD_IS_TRUE_POP_r21 1053 +#define _GUARD_IS_TRUE_POP_r32 1054 +#define _GUARD_ITERATOR_r01 1055 +#define _GUARD_ITERATOR_r11 1056 +#define _GUARD_ITERATOR_r22 1057 +#define _GUARD_ITERATOR_r33 1058 +#define _GUARD_ITER_VIRTUAL_r01 1059 +#define _GUARD_ITER_VIRTUAL_r11 1060 +#define _GUARD_ITER_VIRTUAL_r22 1061 +#define _GUARD_ITER_VIRTUAL_r33 1062 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r03 1063 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r13 1064 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r23 1065 +#define _GUARD_LOAD_SUPER_ATTR_METHOD_r33 1066 +#define _GUARD_NOS_COMPACT_ASCII_r02 1067 +#define _GUARD_NOS_COMPACT_ASCII_r12 1068 +#define _GUARD_NOS_COMPACT_ASCII_r22 1069 +#define _GUARD_NOS_COMPACT_ASCII_r33 1070 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03 1071 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13 1072 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23 1073 +#define _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33 1074 +#define _GUARD_NOS_DICT_SUBSCRIPT_r02 1075 +#define _GUARD_NOS_DICT_SUBSCRIPT_r12 1076 +#define _GUARD_NOS_DICT_SUBSCRIPT_r22 1077 +#define _GUARD_NOS_DICT_SUBSCRIPT_r33 1078 +#define _GUARD_NOS_FLOAT_r02 1079 +#define _GUARD_NOS_FLOAT_r12 1080 +#define _GUARD_NOS_FLOAT_r22 1081 +#define _GUARD_NOS_FLOAT_r33 1082 +#define _GUARD_NOS_INT_r02 1083 +#define _GUARD_NOS_INT_r12 1084 +#define _GUARD_NOS_INT_r22 1085 +#define _GUARD_NOS_INT_r33 1086 +#define _GUARD_NOS_ITER_VIRTUAL_r02 1087 +#define _GUARD_NOS_ITER_VIRTUAL_r12 1088 +#define _GUARD_NOS_ITER_VIRTUAL_r22 1089 +#define _GUARD_NOS_ITER_VIRTUAL_r33 1090 +#define _GUARD_NOS_LIST_r02 1091 +#define _GUARD_NOS_LIST_r12 1092 +#define _GUARD_NOS_LIST_r22 1093 +#define _GUARD_NOS_LIST_r33 1094 +#define _GUARD_NOS_NOT_NULL_r02 1095 +#define _GUARD_NOS_NOT_NULL_r12 1096 +#define _GUARD_NOS_NOT_NULL_r22 1097 +#define _GUARD_NOS_NOT_NULL_r33 1098 +#define _GUARD_NOS_NULL_r02 1099 +#define _GUARD_NOS_NULL_r12 1100 +#define _GUARD_NOS_NULL_r22 1101 +#define _GUARD_NOS_NULL_r33 1102 +#define _GUARD_NOS_OVERFLOWED_r02 1103 +#define _GUARD_NOS_OVERFLOWED_r12 1104 +#define _GUARD_NOS_OVERFLOWED_r22 1105 +#define _GUARD_NOS_OVERFLOWED_r33 1106 +#define _GUARD_NOS_TUPLE_r02 1107 +#define _GUARD_NOS_TUPLE_r12 1108 +#define _GUARD_NOS_TUPLE_r22 1109 +#define _GUARD_NOS_TUPLE_r33 1110 +#define _GUARD_NOS_TYPE_VERSION_r02 1111 +#define _GUARD_NOS_TYPE_VERSION_r12 1112 +#define _GUARD_NOS_TYPE_VERSION_r22 1113 +#define _GUARD_NOS_TYPE_VERSION_r33 1114 +#define _GUARD_NOS_UNICODE_r02 1115 +#define _GUARD_NOS_UNICODE_r12 1116 +#define _GUARD_NOS_UNICODE_r22 1117 +#define _GUARD_NOS_UNICODE_r33 1118 +#define _GUARD_NOT_EXHAUSTED_LIST_r02 1119 +#define _GUARD_NOT_EXHAUSTED_LIST_r12 1120 +#define _GUARD_NOT_EXHAUSTED_LIST_r22 1121 +#define _GUARD_NOT_EXHAUSTED_LIST_r33 1122 +#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1123 +#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1124 +#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1125 +#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1126 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1127 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1128 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1129 +#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1130 +#define _GUARD_THIRD_NULL_r03 1131 +#define _GUARD_THIRD_NULL_r13 1132 +#define _GUARD_THIRD_NULL_r23 1133 +#define _GUARD_THIRD_NULL_r33 1134 +#define _GUARD_TOS_ANY_DICT_r01 1135 +#define _GUARD_TOS_ANY_DICT_r11 1136 +#define _GUARD_TOS_ANY_DICT_r22 1137 +#define _GUARD_TOS_ANY_DICT_r33 1138 +#define _GUARD_TOS_ANY_SET_r01 1139 +#define _GUARD_TOS_ANY_SET_r11 1140 +#define _GUARD_TOS_ANY_SET_r22 1141 +#define _GUARD_TOS_ANY_SET_r33 1142 +#define _GUARD_TOS_DICT_r01 1143 +#define _GUARD_TOS_DICT_r11 1144 +#define _GUARD_TOS_DICT_r22 1145 +#define _GUARD_TOS_DICT_r33 1146 +#define _GUARD_TOS_FLOAT_r01 1147 +#define _GUARD_TOS_FLOAT_r11 1148 +#define _GUARD_TOS_FLOAT_r22 1149 +#define _GUARD_TOS_FLOAT_r33 1150 +#define _GUARD_TOS_FROZENDICT_r01 1151 +#define _GUARD_TOS_FROZENDICT_r11 1152 +#define _GUARD_TOS_FROZENDICT_r22 1153 +#define _GUARD_TOS_FROZENDICT_r33 1154 +#define _GUARD_TOS_FROZENSET_r01 1155 +#define _GUARD_TOS_FROZENSET_r11 1156 +#define _GUARD_TOS_FROZENSET_r22 1157 +#define _GUARD_TOS_FROZENSET_r33 1158 +#define _GUARD_TOS_INT_r01 1159 +#define _GUARD_TOS_INT_r11 1160 +#define _GUARD_TOS_INT_r22 1161 +#define _GUARD_TOS_INT_r33 1162 +#define _GUARD_TOS_IS_NONE_r01 1163 +#define _GUARD_TOS_IS_NONE_r11 1164 +#define _GUARD_TOS_IS_NONE_r22 1165 +#define _GUARD_TOS_IS_NONE_r33 1166 +#define _GUARD_TOS_LIST_r01 1167 +#define _GUARD_TOS_LIST_r11 1168 +#define _GUARD_TOS_LIST_r22 1169 +#define _GUARD_TOS_LIST_r33 1170 +#define _GUARD_TOS_NOT_NULL_r01 1171 +#define _GUARD_TOS_NOT_NULL_r11 1172 +#define _GUARD_TOS_NOT_NULL_r22 1173 +#define _GUARD_TOS_NOT_NULL_r33 1174 +#define _GUARD_TOS_OVERFLOWED_r01 1175 +#define _GUARD_TOS_OVERFLOWED_r11 1176 +#define _GUARD_TOS_OVERFLOWED_r22 1177 +#define _GUARD_TOS_OVERFLOWED_r33 1178 +#define _GUARD_TOS_SET_r01 1179 +#define _GUARD_TOS_SET_r11 1180 +#define _GUARD_TOS_SET_r22 1181 +#define _GUARD_TOS_SET_r33 1182 +#define _GUARD_TOS_SLICE_r01 1183 +#define _GUARD_TOS_SLICE_r11 1184 +#define _GUARD_TOS_SLICE_r22 1185 +#define _GUARD_TOS_SLICE_r33 1186 +#define _GUARD_TOS_TUPLE_r01 1187 +#define _GUARD_TOS_TUPLE_r11 1188 +#define _GUARD_TOS_TUPLE_r22 1189 +#define _GUARD_TOS_TUPLE_r33 1190 +#define _GUARD_TOS_UNICODE_r01 1191 +#define _GUARD_TOS_UNICODE_r11 1192 +#define _GUARD_TOS_UNICODE_r22 1193 +#define _GUARD_TOS_UNICODE_r33 1194 +#define _GUARD_TYPE_r01 1195 +#define _GUARD_TYPE_r11 1196 +#define _GUARD_TYPE_r22 1197 +#define _GUARD_TYPE_r33 1198 +#define _GUARD_TYPE_ITER_r02 1199 +#define _GUARD_TYPE_ITER_r12 1200 +#define _GUARD_TYPE_ITER_r22 1201 +#define _GUARD_TYPE_ITER_r33 1202 +#define _GUARD_TYPE_VERSION_r01 1203 +#define _GUARD_TYPE_VERSION_r11 1204 +#define _GUARD_TYPE_VERSION_r22 1205 +#define _GUARD_TYPE_VERSION_r33 1206 +#define _GUARD_TYPE_VERSION_LOCKED_r01 1207 +#define _GUARD_TYPE_VERSION_LOCKED_r11 1208 +#define _GUARD_TYPE_VERSION_LOCKED_r22 1209 +#define _GUARD_TYPE_VERSION_LOCKED_r33 1210 +#define _HANDLE_PENDING_AND_DEOPT_r00 1211 +#define _HANDLE_PENDING_AND_DEOPT_r10 1212 +#define _HANDLE_PENDING_AND_DEOPT_r20 1213 +#define _HANDLE_PENDING_AND_DEOPT_r30 1214 +#define _IMPORT_FROM_r12 1215 +#define _IMPORT_NAME_r21 1216 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1217 +#define _INIT_CALL_PY_EXACT_ARGS_r01 1218 +#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1219 +#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1220 +#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1221 +#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1222 +#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1223 +#define _INSERT_NULL_r10 1224 +#define _INSTRUMENTED_FOR_ITER_r23 1225 +#define _INSTRUMENTED_INSTRUCTION_r00 1226 +#define _INSTRUMENTED_JUMP_FORWARD_r00 1227 +#define _INSTRUMENTED_JUMP_FORWARD_r11 1228 +#define _INSTRUMENTED_JUMP_FORWARD_r22 1229 +#define _INSTRUMENTED_JUMP_FORWARD_r33 1230 +#define _INSTRUMENTED_LINE_r00 1231 +#define _INSTRUMENTED_NOT_TAKEN_r00 1232 +#define _INSTRUMENTED_NOT_TAKEN_r11 1233 +#define _INSTRUMENTED_NOT_TAKEN_r22 1234 +#define _INSTRUMENTED_NOT_TAKEN_r33 1235 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1236 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1237 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1238 +#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1239 +#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1240 +#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1241 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1242 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1243 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1244 +#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1245 +#define _IS_NONE_r11 1246 +#define _IS_OP_r03 1247 +#define _IS_OP_r13 1248 +#define _IS_OP_r23 1249 +#define _ITER_CHECK_LIST_r02 1250 +#define _ITER_CHECK_LIST_r12 1251 +#define _ITER_CHECK_LIST_r22 1252 +#define _ITER_CHECK_LIST_r33 1253 +#define _ITER_CHECK_RANGE_r02 1254 +#define _ITER_CHECK_RANGE_r12 1255 +#define _ITER_CHECK_RANGE_r22 1256 +#define _ITER_CHECK_RANGE_r33 1257 +#define _ITER_CHECK_TUPLE_r02 1258 +#define _ITER_CHECK_TUPLE_r12 1259 +#define _ITER_CHECK_TUPLE_r22 1260 +#define _ITER_CHECK_TUPLE_r33 1261 +#define _ITER_JUMP_LIST_r02 1262 +#define _ITER_JUMP_LIST_r12 1263 +#define _ITER_JUMP_LIST_r22 1264 +#define _ITER_JUMP_LIST_r33 1265 +#define _ITER_JUMP_RANGE_r02 1266 +#define _ITER_JUMP_RANGE_r12 1267 +#define _ITER_JUMP_RANGE_r22 1268 +#define _ITER_JUMP_RANGE_r33 1269 +#define _ITER_JUMP_TUPLE_r02 1270 +#define _ITER_JUMP_TUPLE_r12 1271 +#define _ITER_JUMP_TUPLE_r22 1272 +#define _ITER_JUMP_TUPLE_r33 1273 +#define _ITER_NEXT_INLINE_r23 1274 +#define _ITER_NEXT_LIST_r23 1275 +#define _ITER_NEXT_LIST_TIER_TWO_r23 1276 +#define _ITER_NEXT_RANGE_r03 1277 +#define _ITER_NEXT_RANGE_r13 1278 +#define _ITER_NEXT_RANGE_r23 1279 +#define _ITER_NEXT_TUPLE_r03 1280 +#define _ITER_NEXT_TUPLE_r13 1281 +#define _ITER_NEXT_TUPLE_r23 1282 +#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1283 +#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1284 +#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1285 +#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1286 +#define _JUMP_TO_TOP_r00 1287 +#define _LIST_APPEND_r10 1288 +#define _LIST_EXTEND_r11 1289 +#define _LOAD_ATTR_r10 1290 +#define _LOAD_ATTR_CLASS_r11 1291 +#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 1292 +#define _LOAD_ATTR_INSTANCE_VALUE_r02 1293 +#define _LOAD_ATTR_INSTANCE_VALUE_r12 1294 +#define _LOAD_ATTR_INSTANCE_VALUE_r23 1295 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1296 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1297 +#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1298 +#define _LOAD_ATTR_METHOD_NO_DICT_r02 1299 +#define _LOAD_ATTR_METHOD_NO_DICT_r12 1300 +#define _LOAD_ATTR_METHOD_NO_DICT_r23 1301 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1302 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1303 +#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1304 +#define _LOAD_ATTR_MODULE_r12 1305 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1306 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1307 +#define _LOAD_ATTR_PROPERTY_FRAME_r01 1308 +#define _LOAD_ATTR_PROPERTY_FRAME_r11 1309 +#define _LOAD_ATTR_PROPERTY_FRAME_r22 1310 +#define _LOAD_ATTR_PROPERTY_FRAME_r33 1311 +#define _LOAD_ATTR_SLOT_r02 1312 +#define _LOAD_ATTR_SLOT_r12 1313 +#define _LOAD_ATTR_SLOT_r23 1314 +#define _LOAD_ATTR_WITH_HINT_r12 1315 +#define _LOAD_BUILD_CLASS_r01 1316 +#define _LOAD_BYTECODE_r00 1317 +#define _LOAD_COMMON_CONSTANT_r01 1318 +#define _LOAD_COMMON_CONSTANT_r12 1319 +#define _LOAD_COMMON_CONSTANT_r23 1320 +#define _LOAD_CONST_r01 1321 +#define _LOAD_CONST_r12 1322 +#define _LOAD_CONST_r23 1323 +#define _LOAD_CONST_INLINE_r01 1324 +#define _LOAD_CONST_INLINE_r12 1325 +#define _LOAD_CONST_INLINE_r23 1326 +#define _LOAD_CONST_INLINE_BORROW_r01 1327 +#define _LOAD_CONST_INLINE_BORROW_r12 1328 +#define _LOAD_CONST_INLINE_BORROW_r23 1329 +#define _LOAD_DEREF_r01 1330 +#define _LOAD_FAST_r01 1331 +#define _LOAD_FAST_r12 1332 +#define _LOAD_FAST_r23 1333 +#define _LOAD_FAST_0_r01 1334 +#define _LOAD_FAST_0_r12 1335 +#define _LOAD_FAST_0_r23 1336 +#define _LOAD_FAST_1_r01 1337 +#define _LOAD_FAST_1_r12 1338 +#define _LOAD_FAST_1_r23 1339 +#define _LOAD_FAST_2_r01 1340 +#define _LOAD_FAST_2_r12 1341 +#define _LOAD_FAST_2_r23 1342 +#define _LOAD_FAST_3_r01 1343 +#define _LOAD_FAST_3_r12 1344 +#define _LOAD_FAST_3_r23 1345 +#define _LOAD_FAST_4_r01 1346 +#define _LOAD_FAST_4_r12 1347 +#define _LOAD_FAST_4_r23 1348 +#define _LOAD_FAST_5_r01 1349 +#define _LOAD_FAST_5_r12 1350 +#define _LOAD_FAST_5_r23 1351 +#define _LOAD_FAST_6_r01 1352 +#define _LOAD_FAST_6_r12 1353 +#define _LOAD_FAST_6_r23 1354 +#define _LOAD_FAST_7_r01 1355 +#define _LOAD_FAST_7_r12 1356 +#define _LOAD_FAST_7_r23 1357 +#define _LOAD_FAST_AND_CLEAR_r01 1358 +#define _LOAD_FAST_AND_CLEAR_r12 1359 +#define _LOAD_FAST_AND_CLEAR_r23 1360 +#define _LOAD_FAST_BORROW_r01 1361 +#define _LOAD_FAST_BORROW_r12 1362 +#define _LOAD_FAST_BORROW_r23 1363 +#define _LOAD_FAST_BORROW_0_r01 1364 +#define _LOAD_FAST_BORROW_0_r12 1365 +#define _LOAD_FAST_BORROW_0_r23 1366 +#define _LOAD_FAST_BORROW_1_r01 1367 +#define _LOAD_FAST_BORROW_1_r12 1368 +#define _LOAD_FAST_BORROW_1_r23 1369 +#define _LOAD_FAST_BORROW_2_r01 1370 +#define _LOAD_FAST_BORROW_2_r12 1371 +#define _LOAD_FAST_BORROW_2_r23 1372 +#define _LOAD_FAST_BORROW_3_r01 1373 +#define _LOAD_FAST_BORROW_3_r12 1374 +#define _LOAD_FAST_BORROW_3_r23 1375 +#define _LOAD_FAST_BORROW_4_r01 1376 +#define _LOAD_FAST_BORROW_4_r12 1377 +#define _LOAD_FAST_BORROW_4_r23 1378 +#define _LOAD_FAST_BORROW_5_r01 1379 +#define _LOAD_FAST_BORROW_5_r12 1380 +#define _LOAD_FAST_BORROW_5_r23 1381 +#define _LOAD_FAST_BORROW_6_r01 1382 +#define _LOAD_FAST_BORROW_6_r12 1383 +#define _LOAD_FAST_BORROW_6_r23 1384 +#define _LOAD_FAST_BORROW_7_r01 1385 +#define _LOAD_FAST_BORROW_7_r12 1386 +#define _LOAD_FAST_BORROW_7_r23 1387 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1388 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1389 +#define _LOAD_FAST_CHECK_r01 1390 +#define _LOAD_FAST_CHECK_r12 1391 +#define _LOAD_FAST_CHECK_r23 1392 +#define _LOAD_FAST_LOAD_FAST_r02 1393 +#define _LOAD_FAST_LOAD_FAST_r13 1394 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1395 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1396 +#define _LOAD_GLOBAL_r00 1397 +#define _LOAD_GLOBAL_BUILTINS_r01 1398 +#define _LOAD_GLOBAL_MODULE_r01 1399 +#define _LOAD_LOCALS_r01 1400 +#define _LOAD_LOCALS_r12 1401 +#define _LOAD_LOCALS_r23 1402 +#define _LOAD_NAME_r01 1403 +#define _LOAD_SMALL_INT_r01 1404 +#define _LOAD_SMALL_INT_r12 1405 +#define _LOAD_SMALL_INT_r23 1406 +#define _LOAD_SMALL_INT_0_r01 1407 +#define _LOAD_SMALL_INT_0_r12 1408 +#define _LOAD_SMALL_INT_0_r23 1409 +#define _LOAD_SMALL_INT_1_r01 1410 +#define _LOAD_SMALL_INT_1_r12 1411 +#define _LOAD_SMALL_INT_1_r23 1412 +#define _LOAD_SMALL_INT_2_r01 1413 +#define _LOAD_SMALL_INT_2_r12 1414 +#define _LOAD_SMALL_INT_2_r23 1415 +#define _LOAD_SMALL_INT_3_r01 1416 +#define _LOAD_SMALL_INT_3_r12 1417 +#define _LOAD_SMALL_INT_3_r23 1418 +#define _LOAD_SPECIAL_r00 1419 +#define _LOAD_SUPER_ATTR_ATTR_r31 1420 +#define _LOAD_SUPER_ATTR_METHOD_r32 1421 +#define _LOCK_OBJECT_r01 1422 +#define _LOCK_OBJECT_r11 1423 +#define _LOCK_OBJECT_r22 1424 +#define _LOCK_OBJECT_r33 1425 +#define _MAKE_CALLARGS_A_TUPLE_r33 1426 +#define _MAKE_CELL_r00 1427 +#define _MAKE_FUNCTION_r12 1428 +#define _MAKE_HEAP_SAFE_r01 1429 +#define _MAKE_HEAP_SAFE_r11 1430 +#define _MAKE_HEAP_SAFE_r22 1431 +#define _MAKE_HEAP_SAFE_r33 1432 +#define _MAKE_WARM_r00 1433 +#define _MAKE_WARM_r11 1434 +#define _MAKE_WARM_r22 1435 +#define _MAKE_WARM_r33 1436 +#define _MAP_ADD_r20 1437 +#define _MATCH_CLASS_r33 1438 +#define _MATCH_KEYS_r23 1439 +#define _MATCH_MAPPING_r02 1440 +#define _MATCH_MAPPING_r12 1441 +#define _MATCH_MAPPING_r23 1442 +#define _MATCH_SEQUENCE_r02 1443 +#define _MATCH_SEQUENCE_r12 1444 +#define _MATCH_SEQUENCE_r23 1445 +#define _MAYBE_EXPAND_METHOD_r00 1446 +#define _MAYBE_EXPAND_METHOD_KW_r11 1447 +#define _MONITOR_CALL_r00 1448 +#define _MONITOR_CALL_KW_r11 1449 +#define _MONITOR_JUMP_BACKWARD_r00 1450 +#define _MONITOR_JUMP_BACKWARD_r11 1451 +#define _MONITOR_JUMP_BACKWARD_r22 1452 +#define _MONITOR_JUMP_BACKWARD_r33 1453 +#define _MONITOR_RESUME_r00 1454 +#define _NOP_r00 1455 +#define _NOP_r11 1456 +#define _NOP_r22 1457 +#define _NOP_r33 1458 +#define _POP_EXCEPT_r10 1459 +#define _POP_ITER_r20 1460 +#define _POP_JUMP_IF_FALSE_r00 1461 +#define _POP_JUMP_IF_FALSE_r10 1462 +#define _POP_JUMP_IF_FALSE_r21 1463 +#define _POP_JUMP_IF_FALSE_r32 1464 +#define _POP_JUMP_IF_TRUE_r00 1465 +#define _POP_JUMP_IF_TRUE_r10 1466 +#define _POP_JUMP_IF_TRUE_r21 1467 +#define _POP_JUMP_IF_TRUE_r32 1468 +#define _POP_TOP_r10 1469 +#define _POP_TOP_FLOAT_r00 1470 +#define _POP_TOP_FLOAT_r10 1471 +#define _POP_TOP_FLOAT_r21 1472 +#define _POP_TOP_FLOAT_r32 1473 +#define _POP_TOP_INT_r00 1474 +#define _POP_TOP_INT_r10 1475 +#define _POP_TOP_INT_r21 1476 +#define _POP_TOP_INT_r32 1477 +#define _POP_TOP_NOP_r00 1478 +#define _POP_TOP_NOP_r10 1479 +#define _POP_TOP_NOP_r21 1480 +#define _POP_TOP_NOP_r32 1481 +#define _POP_TOP_OPARG_r00 1482 +#define _POP_TOP_UNICODE_r00 1483 +#define _POP_TOP_UNICODE_r10 1484 +#define _POP_TOP_UNICODE_r21 1485 +#define _POP_TOP_UNICODE_r32 1486 +#define _PUSH_EXC_INFO_r02 1487 +#define _PUSH_EXC_INFO_r12 1488 +#define _PUSH_EXC_INFO_r23 1489 +#define _PUSH_FRAME_r10 1490 +#define _PUSH_NULL_r01 1491 +#define _PUSH_NULL_r12 1492 +#define _PUSH_NULL_r23 1493 +#define _PUSH_NULL_CONDITIONAL_r00 1494 +#define _PUSH_TAGGED_ZERO_r01 1495 +#define _PUSH_TAGGED_ZERO_r12 1496 +#define _PUSH_TAGGED_ZERO_r23 1497 +#define _PY_FRAME_EX_r31 1498 +#define _PY_FRAME_GENERAL_r01 1499 +#define _PY_FRAME_KW_r11 1500 +#define _REPLACE_WITH_TRUE_r02 1501 +#define _REPLACE_WITH_TRUE_r12 1502 +#define _REPLACE_WITH_TRUE_r23 1503 +#define _RESUME_CHECK_r00 1504 +#define _RESUME_CHECK_r11 1505 +#define _RESUME_CHECK_r22 1506 +#define _RESUME_CHECK_r33 1507 +#define _RETURN_GENERATOR_r01 1508 +#define _RETURN_VALUE_r11 1509 +#define _RROT_3_r03 1510 +#define _RROT_3_r13 1511 +#define _RROT_3_r23 1512 +#define _RROT_3_r33 1513 +#define _SAVE_RETURN_OFFSET_r00 1514 +#define _SAVE_RETURN_OFFSET_r11 1515 +#define _SAVE_RETURN_OFFSET_r22 1516 +#define _SAVE_RETURN_OFFSET_r33 1517 +#define _SEND_ASYNC_GEN_r33 1518 +#define _SEND_ASYNC_GEN_TIER_TWO_r33 1519 +#define _SEND_GEN_FRAME_r33 1520 +#define _SEND_VIRTUAL_r33 1521 +#define _SEND_VIRTUAL_TIER_TWO_r03 1522 +#define _SEND_VIRTUAL_TIER_TWO_r13 1523 +#define _SEND_VIRTUAL_TIER_TWO_r23 1524 +#define _SEND_VIRTUAL_TIER_TWO_r33 1525 +#define _SETUP_ANNOTATIONS_r00 1526 +#define _SET_ADD_r10 1527 +#define _SET_FUNCTION_ATTRIBUTE_r01 1528 +#define _SET_FUNCTION_ATTRIBUTE_r11 1529 +#define _SET_FUNCTION_ATTRIBUTE_r21 1530 +#define _SET_FUNCTION_ATTRIBUTE_r32 1531 +#define _SET_IP_r00 1532 +#define _SET_IP_r11 1533 +#define _SET_IP_r22 1534 +#define _SET_IP_r33 1535 +#define _SET_UPDATE_r11 1536 +#define _SPILL_OR_RELOAD_r01 1537 +#define _SPILL_OR_RELOAD_r02 1538 +#define _SPILL_OR_RELOAD_r03 1539 +#define _SPILL_OR_RELOAD_r10 1540 +#define _SPILL_OR_RELOAD_r12 1541 +#define _SPILL_OR_RELOAD_r13 1542 +#define _SPILL_OR_RELOAD_r20 1543 +#define _SPILL_OR_RELOAD_r21 1544 +#define _SPILL_OR_RELOAD_r23 1545 +#define _SPILL_OR_RELOAD_r30 1546 +#define _SPILL_OR_RELOAD_r31 1547 +#define _SPILL_OR_RELOAD_r32 1548 +#define _START_EXECUTOR_r00 1549 +#define _STORE_ATTR_r20 1550 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1551 +#define _STORE_ATTR_SLOT_r21 1552 +#define _STORE_ATTR_WITH_HINT_r21 1553 +#define _STORE_DEREF_r10 1554 +#define _STORE_FAST_LOAD_FAST_r11 1555 +#define _STORE_FAST_STORE_FAST_r20 1556 +#define _STORE_GLOBAL_r10 1557 +#define _STORE_NAME_r10 1558 +#define _STORE_SLICE_r30 1559 +#define _STORE_SUBSCR_r30 1560 +#define _STORE_SUBSCR_DICT_r31 1561 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1562 +#define _STORE_SUBSCR_LIST_INT_r32 1563 +#define _SWAP_r11 1564 +#define _SWAP_2_r02 1565 +#define _SWAP_2_r12 1566 +#define _SWAP_2_r22 1567 +#define _SWAP_2_r33 1568 +#define _SWAP_3_r03 1569 +#define _SWAP_3_r13 1570 +#define _SWAP_3_r23 1571 +#define _SWAP_3_r33 1572 +#define _SWAP_FAST_r01 1573 +#define _SWAP_FAST_r11 1574 +#define _SWAP_FAST_r22 1575 +#define _SWAP_FAST_r33 1576 +#define _SWAP_FAST_0_r01 1577 +#define _SWAP_FAST_0_r11 1578 +#define _SWAP_FAST_0_r22 1579 +#define _SWAP_FAST_0_r33 1580 +#define _SWAP_FAST_1_r01 1581 +#define _SWAP_FAST_1_r11 1582 +#define _SWAP_FAST_1_r22 1583 +#define _SWAP_FAST_1_r33 1584 +#define _SWAP_FAST_2_r01 1585 +#define _SWAP_FAST_2_r11 1586 +#define _SWAP_FAST_2_r22 1587 +#define _SWAP_FAST_2_r33 1588 +#define _SWAP_FAST_3_r01 1589 +#define _SWAP_FAST_3_r11 1590 +#define _SWAP_FAST_3_r22 1591 +#define _SWAP_FAST_3_r33 1592 +#define _SWAP_FAST_4_r01 1593 +#define _SWAP_FAST_4_r11 1594 +#define _SWAP_FAST_4_r22 1595 +#define _SWAP_FAST_4_r33 1596 +#define _SWAP_FAST_5_r01 1597 +#define _SWAP_FAST_5_r11 1598 +#define _SWAP_FAST_5_r22 1599 +#define _SWAP_FAST_5_r33 1600 +#define _SWAP_FAST_6_r01 1601 +#define _SWAP_FAST_6_r11 1602 +#define _SWAP_FAST_6_r22 1603 +#define _SWAP_FAST_6_r33 1604 +#define _SWAP_FAST_7_r01 1605 +#define _SWAP_FAST_7_r11 1606 +#define _SWAP_FAST_7_r22 1607 +#define _SWAP_FAST_7_r33 1608 +#define _TIER2_RESUME_CHECK_r00 1609 +#define _TIER2_RESUME_CHECK_r11 1610 +#define _TIER2_RESUME_CHECK_r22 1611 +#define _TIER2_RESUME_CHECK_r33 1612 +#define _TO_BOOL_r11 1613 +#define _TO_BOOL_BOOL_r01 1614 +#define _TO_BOOL_BOOL_r11 1615 +#define _TO_BOOL_BOOL_r22 1616 +#define _TO_BOOL_BOOL_r33 1617 +#define _TO_BOOL_INT_r02 1618 +#define _TO_BOOL_INT_r12 1619 +#define _TO_BOOL_INT_r23 1620 +#define _TO_BOOL_LIST_r02 1621 +#define _TO_BOOL_LIST_r12 1622 +#define _TO_BOOL_LIST_r23 1623 +#define _TO_BOOL_NONE_r01 1624 +#define _TO_BOOL_NONE_r11 1625 +#define _TO_BOOL_NONE_r22 1626 +#define _TO_BOOL_NONE_r33 1627 +#define _TO_BOOL_STR_r02 1628 +#define _TO_BOOL_STR_r12 1629 +#define _TO_BOOL_STR_r23 1630 +#define _TRACE_RECORD_r00 1631 +#define _UNARY_INVERT_r12 1632 +#define _UNARY_NEGATIVE_r12 1633 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1634 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1635 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1636 +#define _UNARY_NOT_r01 1637 +#define _UNARY_NOT_r11 1638 +#define _UNARY_NOT_r22 1639 +#define _UNARY_NOT_r33 1640 +#define _UNPACK_EX_r10 1641 +#define _UNPACK_SEQUENCE_r10 1642 +#define _UNPACK_SEQUENCE_LIST_r10 1643 +#define _UNPACK_SEQUENCE_TUPLE_r10 1644 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1645 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1646 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1647 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1648 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1649 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1650 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1651 +#define _WITH_EXCEPT_START_r33 1652 +#define _YIELD_VALUE_r11 1653 +#define MAX_UOP_REGS_ID 1653 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index d6ab67c2e2e86eb..43498c57d8a9d1e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -163,12 +163,17 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_HEAP_SAFE] = 0, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_GUARD_TOS_IS_NONE] = HAS_EXIT_FLAG, + [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, + [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, + [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -262,7 +267,10 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_PUSH_TAGGED_ZERO] = 0, [_GET_ITER_TRAD] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_ITER] = HAS_EXIT_FLAG, + [_ITER_NEXT_INLINE] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_GUARD_TOS_NOT_NULL] = HAS_EXIT_FLAG, [_FOR_ITER_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, @@ -279,7 +287,6 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_EXIT_FLAG, - [_GUARD_KEYS_VERSION] = HAS_EXIT_FLAG, [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, @@ -308,7 +315,6 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_EXIT_FLAG, - [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_GUARD_THIRD_NULL] = HAS_EXIT_FLAG, [_GUARD_CALLABLE_TYPE_1] = HAS_EXIT_FLAG, [_CALL_TYPE_1] = HAS_ARG_FLAG, @@ -361,7 +367,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_FUNCTION_EX_NON_PY_GENERAL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1639,6 +1645,51 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _SEND_GEN_FRAME_r33 }, }, }, + [_GUARD_TOS_IS_NONE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_IS_NONE_r01 }, + { 1, 1, _GUARD_TOS_IS_NONE_r11 }, + { 2, 2, _GUARD_TOS_IS_NONE_r22 }, + { 3, 3, _GUARD_TOS_IS_NONE_r33 }, + }, + }, + [_GUARD_NOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_NOT_NULL_r02 }, + { 2, 1, _GUARD_NOS_NOT_NULL_r12 }, + { 2, 2, _GUARD_NOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_NOS_NOT_NULL_r33 }, + }, + }, + [_SEND_VIRTUAL_TIER_TWO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _SEND_VIRTUAL_TIER_TWO_r03 }, + { 3, 1, _SEND_VIRTUAL_TIER_TWO_r13 }, + { 3, 2, _SEND_VIRTUAL_TIER_TWO_r23 }, + { 3, 3, _SEND_VIRTUAL_TIER_TWO_r33 }, + }, + }, + [_GUARD_3OS_ASYNC_GEN_ASEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_3OS_ASYNC_GEN_ASEND_r03 }, + { 3, 1, _GUARD_3OS_ASYNC_GEN_ASEND_r13 }, + { 3, 2, _GUARD_3OS_ASYNC_GEN_ASEND_r23 }, + { 3, 3, _GUARD_3OS_ASYNC_GEN_ASEND_r33 }, + }, + }, + [_SEND_ASYNC_GEN_TIER_TWO] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _SEND_ASYNC_GEN_TIER_TWO_r33 }, + }, + }, [_YIELD_VALUE] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -2485,6 +2536,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_TYPE_ITER] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_TYPE_ITER_r02 }, + { 2, 1, _GUARD_TYPE_ITER_r12 }, + { 2, 2, _GUARD_TYPE_ITER_r22 }, + { 3, 3, _GUARD_TYPE_ITER_r33 }, + }, + }, + [_ITER_NEXT_INLINE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, [_GUARD_NOS_ITER_VIRTUAL] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -2494,6 +2563,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_NOS_ITER_VIRTUAL_r33 }, }, }, + [_GUARD_TOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_NOT_NULL_r01 }, + { 1, 1, _GUARD_TOS_NOT_NULL_r11 }, + { 2, 2, _GUARD_TOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_TOS_NOT_NULL_r33 }, + }, + }, [_FOR_ITER_VIRTUAL_TIER_TWO] = { .best = { 2, 2, 2, 2 }, .entries = { @@ -2638,15 +2716,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 }, }, }, - [_GUARD_KEYS_VERSION] = { - .best = { 0, 1, 2, 3 }, - .entries = { - { 1, 0, _GUARD_KEYS_VERSION_r01 }, - { 1, 1, _GUARD_KEYS_VERSION_r11 }, - { 2, 2, _GUARD_KEYS_VERSION_r22 }, - { 3, 3, _GUARD_KEYS_VERSION_r33 }, - }, - }, [_LOAD_ATTR_METHOD_WITH_VALUES] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -2899,15 +2968,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_NOS_NULL_r33 }, }, }, - [_GUARD_NOS_NOT_NULL] = { - .best = { 0, 1, 2, 3 }, - .entries = { - { 2, 0, _GUARD_NOS_NOT_NULL_r02 }, - { 2, 1, _GUARD_NOS_NOT_NULL_r12 }, - { 2, 2, _GUARD_NOS_NOT_NULL_r22 }, - { 3, 3, _GUARD_NOS_NOT_NULL_r33 }, - }, - }, [_GUARD_THIRD_NULL] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -3379,7 +3439,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_RETURN_GENERATOR] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _RETURN_GENERATOR_r01 }, + { 1, 1, _RETURN_GENERATOR_r01 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -4232,6 +4292,23 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r11] = _GET_AWAITABLE, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, + [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r22] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r33] = _GUARD_TOS_IS_NONE, + [_GUARD_NOS_NOT_NULL_r02] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r12] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, + [_SEND_VIRTUAL_TIER_TWO_r03] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r13] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r23] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r33] = _SEND_VIRTUAL_TIER_TWO, + [_GUARD_3OS_ASYNC_GEN_ASEND_r03] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_SEND_ASYNC_GEN_TIER_TWO_r33] = _SEND_ASYNC_GEN_TIER_TWO, [_YIELD_VALUE_r11] = _YIELD_VALUE, [_POP_EXCEPT_r10] = _POP_EXCEPT, [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, @@ -4398,10 +4475,19 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_PUSH_TAGGED_ZERO_r23] = _PUSH_TAGGED_ZERO, [_GET_ITER_TRAD_r12] = _GET_ITER_TRAD, [_FOR_ITER_TIER_TWO_r23] = _FOR_ITER_TIER_TWO, + [_GUARD_TYPE_ITER_r02] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r12] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r22] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r33] = _GUARD_TYPE_ITER, + [_ITER_NEXT_INLINE_r23] = _ITER_NEXT_INLINE, [_GUARD_NOS_ITER_VIRTUAL_r02] = _GUARD_NOS_ITER_VIRTUAL, [_GUARD_NOS_ITER_VIRTUAL_r12] = _GUARD_NOS_ITER_VIRTUAL, [_GUARD_NOS_ITER_VIRTUAL_r22] = _GUARD_NOS_ITER_VIRTUAL, [_GUARD_NOS_ITER_VIRTUAL_r33] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_TOS_NOT_NULL_r01] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r11] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r22] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r33] = _GUARD_TOS_NOT_NULL, [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = _FOR_ITER_VIRTUAL_TIER_TWO, [_ITER_CHECK_LIST_r02] = _ITER_CHECK_LIST, [_ITER_CHECK_LIST_r12] = _ITER_CHECK_LIST, @@ -4447,10 +4533,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, - [_GUARD_KEYS_VERSION_r01] = _GUARD_KEYS_VERSION, - [_GUARD_KEYS_VERSION_r11] = _GUARD_KEYS_VERSION, - [_GUARD_KEYS_VERSION_r22] = _GUARD_KEYS_VERSION, - [_GUARD_KEYS_VERSION_r33] = _GUARD_KEYS_VERSION, [_LOAD_ATTR_METHOD_WITH_VALUES_r02] = _LOAD_ATTR_METHOD_WITH_VALUES, [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = _LOAD_ATTR_METHOD_WITH_VALUES, [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = _LOAD_ATTR_METHOD_WITH_VALUES, @@ -4500,10 +4582,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_NOS_NULL_r12] = _GUARD_NOS_NULL, [_GUARD_NOS_NULL_r22] = _GUARD_NOS_NULL, [_GUARD_NOS_NULL_r33] = _GUARD_NOS_NULL, - [_GUARD_NOS_NOT_NULL_r02] = _GUARD_NOS_NOT_NULL, - [_GUARD_NOS_NOT_NULL_r12] = _GUARD_NOS_NOT_NULL, - [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, - [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, [_GUARD_THIRD_NULL_r03] = _GUARD_THIRD_NULL, [_GUARD_THIRD_NULL_r13] = _GUARD_THIRD_NULL, [_GUARD_THIRD_NULL_r23] = _GUARD_THIRD_NULL, @@ -5181,6 +5259,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_ITER_TRAD_r12] = "_GET_ITER_TRAD_r12", [_GET_LEN] = "_GET_LEN", [_GET_LEN_r12] = "_GET_LEN_r12", + [_GUARD_3OS_ASYNC_GEN_ASEND] = "_GUARD_3OS_ASYNC_GEN_ASEND", + [_GUARD_3OS_ASYNC_GEN_ASEND_r03] = "_GUARD_3OS_ASYNC_GEN_ASEND_r03", + [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = "_GUARD_3OS_ASYNC_GEN_ASEND_r13", + [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = "_GUARD_3OS_ASYNC_GEN_ASEND_r23", + [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = "_GUARD_3OS_ASYNC_GEN_ASEND_r33", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", @@ -5376,11 +5459,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_ITER_VIRTUAL_r11] = "_GUARD_ITER_VIRTUAL_r11", [_GUARD_ITER_VIRTUAL_r22] = "_GUARD_ITER_VIRTUAL_r22", [_GUARD_ITER_VIRTUAL_r33] = "_GUARD_ITER_VIRTUAL_r33", - [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", - [_GUARD_KEYS_VERSION_r01] = "_GUARD_KEYS_VERSION_r01", - [_GUARD_KEYS_VERSION_r11] = "_GUARD_KEYS_VERSION_r11", - [_GUARD_KEYS_VERSION_r22] = "_GUARD_KEYS_VERSION_r22", - [_GUARD_KEYS_VERSION_r33] = "_GUARD_KEYS_VERSION_r33", [_GUARD_LOAD_SUPER_ATTR_METHOD] = "_GUARD_LOAD_SUPER_ATTR_METHOD", [_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r03", [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r13", @@ -5506,11 +5584,21 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_INT_r11] = "_GUARD_TOS_INT_r11", [_GUARD_TOS_INT_r22] = "_GUARD_TOS_INT_r22", [_GUARD_TOS_INT_r33] = "_GUARD_TOS_INT_r33", + [_GUARD_TOS_IS_NONE] = "_GUARD_TOS_IS_NONE", + [_GUARD_TOS_IS_NONE_r01] = "_GUARD_TOS_IS_NONE_r01", + [_GUARD_TOS_IS_NONE_r11] = "_GUARD_TOS_IS_NONE_r11", + [_GUARD_TOS_IS_NONE_r22] = "_GUARD_TOS_IS_NONE_r22", + [_GUARD_TOS_IS_NONE_r33] = "_GUARD_TOS_IS_NONE_r33", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", [_GUARD_TOS_LIST_r01] = "_GUARD_TOS_LIST_r01", [_GUARD_TOS_LIST_r11] = "_GUARD_TOS_LIST_r11", [_GUARD_TOS_LIST_r22] = "_GUARD_TOS_LIST_r22", [_GUARD_TOS_LIST_r33] = "_GUARD_TOS_LIST_r33", + [_GUARD_TOS_NOT_NULL] = "_GUARD_TOS_NOT_NULL", + [_GUARD_TOS_NOT_NULL_r01] = "_GUARD_TOS_NOT_NULL_r01", + [_GUARD_TOS_NOT_NULL_r11] = "_GUARD_TOS_NOT_NULL_r11", + [_GUARD_TOS_NOT_NULL_r22] = "_GUARD_TOS_NOT_NULL_r22", + [_GUARD_TOS_NOT_NULL_r33] = "_GUARD_TOS_NOT_NULL_r33", [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", [_GUARD_TOS_OVERFLOWED_r01] = "_GUARD_TOS_OVERFLOWED_r01", [_GUARD_TOS_OVERFLOWED_r11] = "_GUARD_TOS_OVERFLOWED_r11", @@ -5541,6 +5629,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TYPE_r11] = "_GUARD_TYPE_r11", [_GUARD_TYPE_r22] = "_GUARD_TYPE_r22", [_GUARD_TYPE_r33] = "_GUARD_TYPE_r33", + [_GUARD_TYPE_ITER] = "_GUARD_TYPE_ITER", + [_GUARD_TYPE_ITER_r02] = "_GUARD_TYPE_ITER_r02", + [_GUARD_TYPE_ITER_r12] = "_GUARD_TYPE_ITER_r12", + [_GUARD_TYPE_ITER_r22] = "_GUARD_TYPE_ITER_r22", + [_GUARD_TYPE_ITER_r33] = "_GUARD_TYPE_ITER_r33", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_GUARD_TYPE_VERSION_r01] = "_GUARD_TYPE_VERSION_r01", [_GUARD_TYPE_VERSION_r11] = "_GUARD_TYPE_VERSION_r11", @@ -5597,6 +5690,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_ITER_CHECK_TUPLE_r12] = "_ITER_CHECK_TUPLE_r12", [_ITER_CHECK_TUPLE_r22] = "_ITER_CHECK_TUPLE_r22", [_ITER_CHECK_TUPLE_r33] = "_ITER_CHECK_TUPLE_r33", + [_ITER_NEXT_INLINE] = "_ITER_NEXT_INLINE", + [_ITER_NEXT_INLINE_r23] = "_ITER_NEXT_INLINE_r23", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", [_ITER_NEXT_LIST_TIER_TWO_r23] = "_ITER_NEXT_LIST_TIER_TWO_r23", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", @@ -5920,8 +6015,15 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SAVE_RETURN_OFFSET_r11] = "_SAVE_RETURN_OFFSET_r11", [_SAVE_RETURN_OFFSET_r22] = "_SAVE_RETURN_OFFSET_r22", [_SAVE_RETURN_OFFSET_r33] = "_SAVE_RETURN_OFFSET_r33", + [_SEND_ASYNC_GEN_TIER_TWO] = "_SEND_ASYNC_GEN_TIER_TWO", + [_SEND_ASYNC_GEN_TIER_TWO_r33] = "_SEND_ASYNC_GEN_TIER_TWO_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", + [_SEND_VIRTUAL_TIER_TWO] = "_SEND_VIRTUAL_TIER_TWO", + [_SEND_VIRTUAL_TIER_TWO_r03] = "_SEND_VIRTUAL_TIER_TWO_r03", + [_SEND_VIRTUAL_TIER_TWO_r13] = "_SEND_VIRTUAL_TIER_TWO_r13", + [_SEND_VIRTUAL_TIER_TWO_r23] = "_SEND_VIRTUAL_TIER_TWO_r23", + [_SEND_VIRTUAL_TIER_TWO_r33] = "_SEND_VIRTUAL_TIER_TWO_r33", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", [_SETUP_ANNOTATIONS_r00] = "_SETUP_ANNOTATIONS_r00", [_SET_ADD] = "_SET_ADD", @@ -6369,6 +6471,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _SEND_GEN_FRAME: return 1; + case _GUARD_TOS_IS_NONE: + return 0; + case _GUARD_NOS_NOT_NULL: + return 0; + case _SEND_VIRTUAL_TIER_TWO: + return 1; + case _GUARD_3OS_ASYNC_GEN_ASEND: + return 0; + case _SEND_ASYNC_GEN_TIER_TWO: + return 3; case _YIELD_VALUE: return 1; case _POP_EXCEPT: @@ -6557,8 +6669,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _FOR_ITER_TIER_TWO: return 0; + case _GUARD_TYPE_ITER: + return 0; + case _ITER_NEXT_INLINE: + return 0; case _GUARD_NOS_ITER_VIRTUAL: return 0; + case _GUARD_TOS_NOT_NULL: + return 0; case _FOR_ITER_VIRTUAL_TIER_TWO: return 0; case _ITER_CHECK_LIST: @@ -6591,8 +6709,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: return 0; - case _GUARD_KEYS_VERSION: - return 0; case _LOAD_ATTR_METHOD_WITH_VALUES: return 1; case _LOAD_ATTR_METHOD_NO_DICT: @@ -6649,8 +6765,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_NOS_NULL: return 0; - case _GUARD_NOS_NOT_NULL: - return 0; case _GUARD_THIRD_NULL: return 0; case _GUARD_CALLABLE_TYPE_1: diff --git a/Include/moduleobject.h b/Include/moduleobject.h index c2fb1f85165f7d2..88c66672ff164a8 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -73,31 +73,6 @@ struct PyModuleDef_Slot { void *value; }; -#define Py_mod_create 1 -#define Py_mod_exec 2 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 -# define Py_mod_multiple_interpreters 3 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -# define Py_mod_gil 4 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) -# define Py_mod_abi 5 -# define Py_mod_name 6 -# define Py_mod_doc 7 -# define Py_mod_state_size 8 -# define Py_mod_methods 9 -# define Py_mod_state_traverse 10 -# define Py_mod_state_clear 11 -# define Py_mod_state_free 12 -# define Py_mod_token 13 -#endif - - -#ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 13 -#endif - #endif /* New in 3.5 */ /* for Py_mod_multiple_interpreters: */ @@ -120,7 +95,7 @@ PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) -PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec); PyAPI_FUNC(int) PyModule_Exec(PyObject *module); PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *module, Py_ssize_t *result); diff --git a/Include/object.h b/Include/object.h index 043be0260b1aec4..20c2dab4401fef0 100644 --- a/Include/object.h +++ b/Include/object.h @@ -364,6 +364,9 @@ PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); PyAPI_FUNC(int) PyType_GetBaseByToken(PyTypeObject *, void *, PyTypeObject **); #define Py_TP_USE_SPEC NULL #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyType_FromSlots(struct PySlot *slots); +#endif /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 53a7eaf0de57ebf..758542720acf317 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -204,21 +204,23 @@ extern "C" { #define LOAD_SUPER_ATTR_METHOD 199 #define RESUME_CHECK 200 #define RESUME_CHECK_JIT 201 -#define SEND_GEN 202 -#define STORE_ATTR_INSTANCE_VALUE 203 -#define STORE_ATTR_SLOT 204 -#define STORE_ATTR_WITH_HINT 205 -#define STORE_SUBSCR_DICT 206 -#define STORE_SUBSCR_LIST_INT 207 -#define TO_BOOL_ALWAYS_TRUE 208 -#define TO_BOOL_BOOL 209 -#define TO_BOOL_INT 210 -#define TO_BOOL_LIST 211 -#define TO_BOOL_NONE 212 -#define TO_BOOL_STR 213 -#define UNPACK_SEQUENCE_LIST 214 -#define UNPACK_SEQUENCE_TUPLE 215 -#define UNPACK_SEQUENCE_TWO_TUPLE 216 +#define SEND_ASYNC_GEN 202 +#define SEND_GEN 203 +#define SEND_VIRTUAL 204 +#define STORE_ATTR_INSTANCE_VALUE 205 +#define STORE_ATTR_SLOT 206 +#define STORE_ATTR_WITH_HINT 207 +#define STORE_SUBSCR_DICT 208 +#define STORE_SUBSCR_LIST_INT 209 +#define TO_BOOL_ALWAYS_TRUE 210 +#define TO_BOOL_BOOL 211 +#define TO_BOOL_INT 212 +#define TO_BOOL_LIST 213 +#define TO_BOOL_NONE 214 +#define TO_BOOL_STR 215 +#define UNPACK_SEQUENCE_LIST 216 +#define UNPACK_SEQUENCE_TUPLE 217 +#define UNPACK_SEQUENCE_TWO_TUPLE 218 #define INSTRUMENTED_END_FOR 233 #define INSTRUMENTED_POP_ITER 234 #define INSTRUMENTED_END_SEND 235 diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 974246f896e10b1..d82d52584069413 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -21,13 +21,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 15 +#define PY_MINOR_VERSION 16 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 8 +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.15.0a8+" +#define PY_VERSION "3.16.0a0" /*--end constants--*/ diff --git a/Include/py_curses.h b/Include/py_curses.h index 0948aabedd49939..3d2ca278f809cb2 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -80,8 +80,18 @@ typedef struct PyCursesWindowObject { WINDOW *win; char *encoding; struct PyCursesWindowObject *orig; + PyObject *screen; /* the screen the window belongs to, or NULL, + kept alive for the lifetime of the window */ } PyCursesWindowObject; +typedef struct { + PyObject_HEAD + SCREEN *screen; /* NULL after the screen has been deleted */ + FILE *outfp; /* owned output stream, or NULL */ + FILE *infp; /* owned input stream, or NULL */ + PyObject *stdscr; /* the screen's standard window, or NULL */ +} PyCursesScreenObject; + #define PyCurses_CAPSULE_NAME "_curses._C_API" diff --git a/Include/pyabi.h b/Include/pyabi.h index 8c4ae281a43faf3..21a6ab0c1ee6ea2 100644 --- a/Include/pyabi.h +++ b/Include/pyabi.h @@ -55,6 +55,8 @@ * * (Don't use Py_TARGET_ABI3T directly. It's currently only used to set these * 2 macros, and defined for users' convenience.) + * + * This logic is currently partially duplicated in PC/pyconfig.h. */ #if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ && !defined(Py_TARGET_ABI3T) diff --git a/Include/pyexpat.h b/Include/pyexpat.h index f523f8bb273983a..a676e16a7a457ea 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -62,6 +62,9 @@ struct PyExpat_CAPI XML_Parser parser, unsigned long long activationThresholdBytes); XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)( XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.8.0 */ + XML_Bool (*SetHashSalt16Bytes)( + XML_Parser parser, const uint8_t entropy[16]); /* always add new stuff to the end! */ }; diff --git a/Include/pystate.h b/Include/pystate.h index 727b8fbfffe0e67..8dad748238f4f39 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -120,6 +120,29 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); +/* PEP 788 -- Protection against interpreter finalization */ + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) + +typedef struct PyInterpreterGuard PyInterpreterGuard; +typedef struct PyInterpreterView PyInterpreterView; + +typedef void PyThreadStateToken; + +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterGuard_Close(PyInterpreterGuard *guard); +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromView(PyInterpreterView *view); + +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterView_Close(PyInterpreterView *view); +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromMain(void); + +PyAPI_FUNC(PyThreadStateToken *) PyThreadState_Ensure(PyInterpreterGuard *guard); +PyAPI_FUNC(PyThreadStateToken *) PyThreadState_EnsureFromView(PyInterpreterView *view); +PyAPI_FUNC(void) PyThreadState_Release(PyThreadStateToken *tstate); + +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" diff --git a/Include/pytypedefs.h b/Include/pytypedefs.h index e78ed56a3b67cd1..1d0b4e3e85ef74a 100644 --- a/Include/pytypedefs.h +++ b/Include/pytypedefs.h @@ -14,6 +14,7 @@ typedef struct PyModuleDef_Slot PyModuleDef_Slot; typedef struct PyMethodDef PyMethodDef; typedef struct PyGetSetDef PyGetSetDef; typedef struct PyMemberDef PyMemberDef; +typedef struct PySlot PySlot; typedef struct _object PyObject; typedef struct _longobject PyLongObject; diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 00c70a6e911b417..9d6a16da95fe2f5 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -45,9 +45,9 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, #endif #ifndef Py_LIMITED_API -# define Py_CPYTHON_SLICEOBJECT_H +# define _Py_CPYTHON_SLICEOBJECT_H # include "cpython/sliceobject.h" -# undef Py_CPYTHON_SLICEOBJECT_H +# undef _Py_CPYTHON_SLICEOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/slots.h b/Include/slots.h new file mode 100644 index 000000000000000..4bf7bda0208a8df --- /dev/null +++ b/Include/slots.h @@ -0,0 +1,56 @@ +#ifndef _Py_HAVE_SLOTS_H +#define _Py_HAVE_SLOTS_H + +typedef void (*_Py_funcptr_t)(void); + +struct PySlot { + uint16_t sl_id; + uint16_t sl_flags; + _Py_ANONYMOUS union { + uint32_t sl_reserved; // must be 0 + }; + _Py_ANONYMOUS union { + void *sl_ptr; + _Py_funcptr_t sl_func; + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; +}; + +#define PySlot_OPTIONAL 0x0001 +#define PySlot_STATIC 0x0002 +#define PySlot_INTPTR 0x0004 + +#define Py_slot_invalid 0xffff + +#define PySlot_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_INTPTR, .sl_ptr=(void*)(VALUE)} + +#define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=(NAME), .sl_func=(_Py_funcptr_t)(VALUE)} + +#define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=(NAME), .sl_size=(Py_ssize_t)(VALUE)} + +#define PySlot_INT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_int64=(int64_t)(VALUE)} + +#define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_uint64=(uint64_t)(VALUE)} + +#define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + +#define PySlot_END {0} + + +// Macros without designated initializers (for C++11 and below): + +#define PySlot_PTR(NAME, VALUE) \ + {(NAME), PySlot_INTPTR, {0}, {(void*)(VALUE)}} + +#define PySlot_PTR_STATIC(NAME, VALUE) \ + {(NAME), PySlot_INTPTR | PySlot_STATIC, {0}, {(void*)(VALUE)}} + +#endif // _Py_HAVE_SLOTS_H diff --git a/Include/typeslots.h b/Include/slots_generated.h similarity index 60% rename from Include/typeslots.h rename to Include/slots_generated.h index a7f3017ec02e92f..42edd3ad4c69ffd 100644 --- a/Include/typeslots.h +++ b/Include/slots_generated.h @@ -1,8 +1,15 @@ -/* Do not renumber the file; these numbers are part of the stable ABI. */ -#define Py_bf_getbuffer 1 -#define Py_bf_releasebuffer 2 -#define Py_mp_ass_subscript 3 -#define Py_mp_length 4 +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_SLOTS_GENERATED_H +#define _PY_HAVE_SLOTS_GENERATED_H + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) NEW +#else +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) OLD +#endif + +#define Py_slot_end 0 #define Py_mp_subscript 5 #define Py_nb_absolute 6 #define Py_nb_add 7 @@ -78,19 +85,37 @@ #define Py_am_await 77 #define Py_am_aiter 78 #define Py_am_anext 79 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ #define Py_tp_finalize 80 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 -/* New in 3.10 */ #define Py_am_send 81 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 -/* New in 3.14 */ #define Py_tp_vectorcall 82 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 -/* New in 3.14 */ #define Py_tp_token 83 -#endif +#define Py_mod_create _Py_SLOT_COMPAT_VALUE(1, 84) +#define Py_mod_exec _Py_SLOT_COMPAT_VALUE(2, 85) +#define Py_mod_multiple_interpreters _Py_SLOT_COMPAT_VALUE(3, 86) +#define Py_mod_gil _Py_SLOT_COMPAT_VALUE(4, 87) +#define Py_bf_getbuffer _Py_SLOT_COMPAT_VALUE(1, 88) +#define Py_bf_releasebuffer _Py_SLOT_COMPAT_VALUE(2, 89) +#define Py_mp_ass_subscript _Py_SLOT_COMPAT_VALUE(3, 90) +#define Py_mp_length _Py_SLOT_COMPAT_VALUE(4, 91) +#define Py_slot_subslots 92 +#define Py_tp_slots 93 +#define Py_mod_slots 94 +#define Py_tp_name 95 +#define Py_tp_basicsize 96 +#define Py_tp_extra_basicsize 97 +#define Py_tp_itemsize 98 +#define Py_tp_flags 99 +#define Py_mod_name 100 +#define Py_mod_doc 101 +#define Py_mod_state_size 102 +#define Py_mod_methods 103 +#define Py_mod_state_traverse 104 +#define Py_mod_state_clear 105 +#define Py_mod_state_free 106 +#define Py_tp_metaclass 107 +#define Py_tp_module 108 +#define Py_mod_abi 109 +#define Py_mod_token 110 + +#define _Py_slot_COUNT 111 +#endif /* _PY_HAVE_SLOTS_GENERATED_H */ diff --git a/Include/structseq.h b/Include/structseq.h index e52d6188030af9d..e5da785f13d46b8 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -29,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -# define Py_CPYTHON_STRUCTSEQ_H +# define _Py_CPYTHON_STRUCTSEQ_H # include "cpython/structseq.h" -# undef Py_CPYTHON_STRUCTSEQ_H +# undef _Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 742af5efcf5ce50..9ed4d0eb65a0bdd 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -359,19 +359,19 @@ in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instruction These are similar to bytecode, but in some cases they are more abstract, and are resolved later into actual bytecode. The construction of this instruction sequence is handled by several functions that break the task down by various AST node types. -The functions are all named `compiler_visit_{xx}` where *xx* is the name of the node +The functions are all named `codegen_visit_{xx}` where *xx* is the name of the node type (such as `stmt`, `expr`, etc.). Each function receives a `struct compiler *` and `{xx}_ty` where *xx* is the AST node type. Typically these functions consist of a large 'switch' statement, branching based on the kind of node type passed to it. Simple things are handled inline in the 'switch' statement with more complex transformations farmed out to other -functions named `compiler_{xx}` with *xx* being a descriptive name of what is +functions named `codegen_{xx}` with *xx* being a descriptive name of what is being handled. When transforming an arbitrary AST node, use the `VISIT()` macro. -The appropriate `compiler_visit_{xx}` function is called, based on the value +The appropriate `codegen_visit_{xx}` function is called, based on the value passed in for (so `VISIT({c}, expr, {node})` calls -`compiler_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, +`codegen_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, but is called on AST node sequences (those values that were created as arguments to a node that used the '*' modifier). @@ -414,8 +414,8 @@ which is added at the end of a function is not associated with any line in the source code. There are several helper functions that will emit pseudo-instructions -and are named `compiler_{xx}()` where *xx* is what the function helps -with (`list`, `boolop`, etc.). A rather useful one is `compiler_nameop()`. +and are named `codegen_{xx}()` where *xx* is what the function helps +with (`list`, `boolop`, etc.). A rather useful one is `codegen_nameop()`. This function looks up the scope of a variable and, based on the expression context, emits the proper opcode to load, store, or delete the variable. diff --git a/InternalDocs/qsbr.md b/InternalDocs/qsbr.md index 1c4a79a7b44436a..d511396fdab6452 100644 --- a/InternalDocs/qsbr.md +++ b/InternalDocs/qsbr.md @@ -101,15 +101,39 @@ Periodically, a polling mechanism processes this deferred-free list: To reduce memory contention from frequent updates to the global `wr_seq`, its advancement is sometimes deferred. Instead of incrementing `wr_seq` on every -reclamation request, each thread tracks its number of deferrals locally. Once -the deferral count reaches a limit (QSBR_DEFERRED_LIMIT, currently 10), the -thread advances the global `wr_seq` and resets its local count. - -When an object is added to the deferred-free list, its qsbr_goal is set to -`wr_seq` + 2. By setting the goal to the next sequence value, we ensure it's safe -to defer the global counter advancement. This optimization improves runtime -speed but may increase peak memory usage by slightly delaying when memory can -be reclaimed. +reclamation request, the object's qsbr_goal is set to `wr_seq` + 2 (the value +the counter *would* take on its next advance) without actually advancing the +global counter. This is safe because the goal still corresponds to a future +sequence value that no thread has yet observed as quiescent. + +Whether to actually advance `wr_seq` is decided per request, based on how +much memory and how many items the calling thread has already deferred since +its last advance: + +* For deferred object frees (`_PyMem_FreeDelayed`), the thread tracks both a + count (`deferred_count`) and an estimate of the held memory + (`deferred_memory`). The global `wr_seq` is advanced when the freed block + is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred + memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT` + (127, sized so a chunk of work items is processed before it overflows). + Crossing any of these thresholds also sets a per-thread `should_process` + flag, signalling that the deferred-free list should be drained. + +* For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory` + and advances `wr_seq` when either the individual page or the accumulated + page memory exceeds `QSBR_PAGE_MEM_LIMIT` (4096 * 20 bytes). Advancing + promptly here matters because a held page cannot be reused for a different + size class or by a different thread. + +Processing of the deferred-free list normally happens from the eval breaker +(rather than from inside `_PyMem_FreeDelayed`), which gives the global +`rd_seq` a better chance to have advanced far enough that items can actually +be freed. `_PyMem_ProcessDelayed` is still called from the free path as a +safety valve when a work-item chunk fills up. + +This optimization improves runtime speed but may increase peak memory usage +by slightly delaying when memory can be reclaimed; the size-based thresholds +above bound that extra memory. ## Limitations diff --git a/Lib/.ruff.toml b/Lib/.ruff.toml new file mode 100644 index 000000000000000..c9ae9b95641b318 --- /dev/null +++ b/Lib/.ruff.toml @@ -0,0 +1,21 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +# Unlike Tools/, stdlib can use newer syntax than PYTHON_FOR_REGEN +target-version = "py315" + +[lint] +select = [ + "F401", # Unused import +] + +[lint.per-file-ignores] +"ctypes/__init__.py" = ["F401"] # Re-exports from _ctypes +"ensurepip/__init__.py" = ["F401"] # `import zlib` availability check +"idlelib/idle_test/htest.py" = ["F401"] # Import for Windows DPI side effect +"idlelib/idle_test/test_iomenu.py" = ["F401"] # Imports checked for existence +"importlib/_abc.py" = ["F401"] # Bootstrap-sensitive _bootstrap import +"importlib/machinery.py" = ["F401"] # NamespacePath re-export +"importlib/metadata/__init__.py" = ["F401"] # Synced from importlib_metadata +"profiling/sampling/sample.py" = ["F401"] # Re-exports PROFILING_MODE_* constants +"ssl.py" = ["F401"] # Re-exports from _ssl +"warnings.py" = ["F401"] # Re-exports from _py_warnings diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 23cc6d8faae2dac..0e1d8ccf44a4351 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -461,8 +461,8 @@ def __subclasshook__(cls, C): class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. - This sets ``__args__`` to a tuple containing the flattened ``argtypes`` - followed by ``resulttype``. + This sets ``__args__`` to a tuple containing the flattened + ``argtypes`` followed by ``resulttype``. Example: ``Callable[[int, str], float]`` sets ``__args__`` to ``(int, str, float)``. @@ -928,8 +928,9 @@ def __delitem__(self, key): __marker = object() def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise + KeyError is raised. ''' try: value = self[key] @@ -963,9 +964,12 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] - If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v - In either case, this is followed by: for k, v in F.items(): D[k] = v + If E present and has a .keys() method, does: + for k in E.keys(): D[k] = E[k] + If E present and lacks .keys() method, does: + for (k, v) in E: D[k] = v + In either case, this is followed by: + for k, v in F.items(): D[k] = v ''' if isinstance(other, Mapping): for key in other: @@ -1030,8 +1034,8 @@ def __reversed__(self): yield self[i] def index(self, value, start=0, stop=None): - '''S.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. + '''S.index(value, [start, [stop]]) -> integer -- return first index of + value. Raises ValueError if the value is not present. Supporting start and stop arguments is optional, but recommended. @@ -1139,15 +1143,16 @@ def reverse(self): self[i], self[n-i-1] = self[n-i-1], self[i] def extend(self, values): - 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + """S.extend(iterable) -- extend sequence by appending elements from the + iterable""" if values is self: values = list(values) for v in values: self.append(v) def pop(self, index=-1): - '''S.pop([index]) -> item -- remove and return item at index (default last). - Raise IndexError if list is empty or index is out of range. + '''S.pop([index]) -> item -- remove and return item at index (default + last). Raise IndexError if list is empty or index is out of range. ''' v = self[index] del self[index] diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 5e0c0124e597b89..27eb7f13baca971 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -4,14 +4,12 @@ from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field, Field +lazy from typing import IO, Literal, Self, ClassVar COLORIZE = True - -# types -if False: - from typing import IO, Literal, Self, ClassVar - _theme: Theme +_theme: Theme +type BackgroundStyle = Literal["dark", "light"] class ANSIColors: @@ -319,7 +317,7 @@ class LiveProfiler(ThemeSection): medal_bronze_fg: int = CursesColors.GREEN # Background style: 'dark' or 'light' - background_style: Literal["dark", "light"] = "dark" + background_style: BackgroundStyle = "dark" LiveProfilerLight = LiveProfiler( diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 6b5357a31511905..183d0af30acf438 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -39,6 +39,8 @@ ), SEND=( "SEND_GEN", + "SEND_VIRTUAL", + "SEND_ASYNC_GEN", ), UNPACK_SEQUENCE=( "UNPACK_SEQUENCE_TWO_TUPLE", @@ -205,21 +207,23 @@ LOAD_SUPER_ATTR_METHOD=199, RESUME_CHECK=200, RESUME_CHECK_JIT=201, - SEND_GEN=202, - STORE_ATTR_INSTANCE_VALUE=203, - STORE_ATTR_SLOT=204, - STORE_ATTR_WITH_HINT=205, - STORE_SUBSCR_DICT=206, - STORE_SUBSCR_LIST_INT=207, - TO_BOOL_ALWAYS_TRUE=208, - TO_BOOL_BOOL=209, - TO_BOOL_INT=210, - TO_BOOL_LIST=211, - TO_BOOL_NONE=212, - TO_BOOL_STR=213, - UNPACK_SEQUENCE_LIST=214, - UNPACK_SEQUENCE_TUPLE=215, - UNPACK_SEQUENCE_TWO_TUPLE=216, + SEND_ASYNC_GEN=202, + SEND_GEN=203, + SEND_VIRTUAL=204, + STORE_ATTR_INSTANCE_VALUE=205, + STORE_ATTR_SLOT=206, + STORE_ATTR_WITH_HINT=207, + STORE_SUBSCR_DICT=208, + STORE_SUBSCR_LIST_INT=209, + TO_BOOL_ALWAYS_TRUE=210, + TO_BOOL_BOOL=211, + TO_BOOL_INT=212, + TO_BOOL_LIST=213, + TO_BOOL_NONE=214, + TO_BOOL_STR=215, + UNPACK_SEQUENCE_LIST=216, + UNPACK_SEQUENCE_TUPLE=217, + UNPACK_SEQUENCE_TWO_TUPLE=218, ) opmap = frozendict( diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b6d68f2372850a7..db4ea8d30c7064f 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -55,7 +55,7 @@ def _days_before_year(year): def _days_in_month(year, month): "year, month -> number of days in that month in that year." - assert 1 <= month <= 12, month + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] @@ -358,7 +358,8 @@ def _find_isoformat_datetime_separator(dtstr): def _parse_isoformat_date(dtstr): # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10, # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator - assert len(dtstr) in (7, 8, 10) + if len(dtstr) not in (7, 8, 10): + raise ValueError("Invalid isoformat string") year = int(dtstr[0:4]) has_sep = dtstr[4] == '-' @@ -1987,7 +1988,7 @@ def fromisoformat(cls, date_string): if became_next_day: year, month, day = date_components # Only wrap day/month when it was previously valid - if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + if 1 <= month <= 12 and day <= (days_in_month := _days_in_month(year, month)): # Calculate midnight of the next day day += 1 if day > days_in_month: diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index ef889ea0cc834c1..8c0afd14d616e85 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -107,8 +107,8 @@ class DecimalException(ArithmeticError): anything, though. handle -- Called when context._raise_error is called and the - trap_enabler is not set. First argument is self, second is the - context. More arguments can be given, those being after + trap_enabler is not set. First argument is self, second is + the context. More arguments can be given, those being after the explanation in _raise_error (For example, context._raise_error(NewError, '(-x)!', self._sign) would call NewError().handle(context, self._sign).) @@ -225,11 +225,12 @@ class InvalidContext(InvalidOperation): """Invalid context. Unknown rounding, for example. This occurs and signals invalid-operation if an invalid context was - detected during an operation. This can occur if contexts are not checked - on creation and either the precision exceeds the capability of the - underlying concrete representation or an unknown or unsupported rounding - was specified. These aspects of the context need only be checked when - the values are required to be used. The result is [0,qNaN]. + detected during an operation. This can occur if contexts are not + checked on creation and either the precision exceeds the capability of + the underlying concrete representation or an unknown or unsupported + rounding was specified. These aspects of the context need only be + checked when the values are required to be used. The result is + [0,qNaN]. """ def handle(self, context, *args): @@ -322,8 +323,9 @@ class FloatOperation(DecimalException, TypeError): Decimal.from_float() or context.create_decimal_from_float() do not set the flag. - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise FloatOperation. + Otherwise (the signal is trapped), only equality comparisons and + explicit conversions are silent. All other mixed operations raise + FloatOperation. """ # List of public traps and flags @@ -2901,8 +2903,8 @@ def compare_total(self, other, context=None): """Compares self to other using the abstract representations. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. """ other = _convert_other(other, raiseit=True) @@ -2973,7 +2975,8 @@ def compare_total(self, other, context=None): def compare_total_mag(self, other, context=None): """Compares self to other using abstract repr., ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ other = _convert_other(other, raiseit=True) @@ -4110,9 +4113,9 @@ def create_decimal_from_float(self, f): def abs(self, a): """Returns the absolute value of the operand. - If the operand is negative, the result is the same as using the minus - operation on the operand. Otherwise, the result is the same as using - the plus operation on the operand. + If the operand is negative, the result is the same as using the + minus operation on the operand. Otherwise, the result is the same + as using the plus operation on the operand. >>> ExtendedContext.abs(Decimal('2.1')) Decimal('2.1') @@ -4168,16 +4171,17 @@ def canonical(self, a): def compare(self, a, b): """Compares values numerically. - If the signs of the operands differ, a value representing each operand - ('-1' if the operand is less than zero, '0' if the operand is zero or - negative zero, or '1' if the operand is greater than zero) is used in - place of that operand for the comparison instead of the actual - operand. + If the signs of the operands differ, a value representing each + operand ('-1' if the operand is less than zero, '0' if the operand + is zero or negative zero, or '1' if the operand is greater than + zero) is used in place of that operand for the comparison instead of + the actual operand. - The comparison is then effected by subtracting the second operand from - the first and then returning a value according to the result of the - subtraction: '-1' if the result is less than zero, '0' if the result is - zero or negative zero, or '1' if the result is greater than zero. + The comparison is then effected by subtracting the second operand + from the first and then returning a value according to the result of + the subtraction: '-1' if the result is less than zero, '0' if the + result is zero or negative zero, or '1' if the result is greater + than zero. >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) Decimal('-1') @@ -4240,8 +4244,8 @@ def compare_total(self, a, b): """Compares two operands using their abstract representation. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) Decimal('-1') @@ -4268,7 +4272,8 @@ def compare_total(self, a, b): def compare_total_mag(self, a, b): """Compares two operands using their abstract representation ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ a = _convert_other(a, raiseit=True) return a.compare_total_mag(b) @@ -4926,8 +4931,8 @@ def multiply(self, a, b): If either operand is a special value then the general rules apply. Otherwise, the operands are multiplied together - ('long multiplication'), resulting in a number which may be as long as - the sum of the lengths of the two operands. + ('long multiplication'), resulting in a number which may be as long + as the sum of the lengths of the two operands. >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) Decimal('3.60') @@ -5203,19 +5208,19 @@ def quantize(self, a, b): """Returns a value equal to 'a' (rounded), having the exponent of 'b'. The coefficient of the result is derived from that of the left-hand - operand. It may be rounded using the current rounding setting (if the - exponent is being increased), multiplied by a positive power of ten (if - the exponent is being decreased), or is unchanged (if the exponent is - already equal to that of the right-hand operand). + operand. It may be rounded using the current rounding setting (if + the exponent is being increased), multiplied by a positive power of + ten (if the exponent is being decreased), or is unchanged (if the + exponent is already equal to that of the right-hand operand). Unlike other operations, if the length of the coefficient after the quantize operation would be greater than precision then an Invalid - operation condition is raised. This guarantees that, unless there is - an error condition, the exponent of the result of a quantize is always - equal to that of the right-hand operand. + operation condition is raised. This guarantees that, unless there + is an error condition, the exponent of the result of a quantize is + always equal to that of the right-hand operand. - Also unlike other operations, quantize will never raise Underflow, even - if the result is subnormal and inexact. + Also unlike other operations, quantize will never raise Underflow, + even if the result is subnormal and inexact. >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) Decimal('2.170') @@ -5269,13 +5274,13 @@ def remainder(self, a, b): """Returns the remainder from integer division. The result is the residue of the dividend after the operation of - calculating integer division as described for divide-integer, rounded - to precision digits if necessary. The sign of the result, if - non-zero, is the same as that of the original dividend. + calculating integer division as described for divide-integer, + rounded to precision digits if necessary. The sign of the result, + if non-zero, is the same as that of the original dividend. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) Decimal('2.1') @@ -5309,9 +5314,9 @@ def remainder_near(self, a, b): is chosen). If the result is equal to 0 then its sign will be the sign of a. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) Decimal('-0.9') @@ -5369,8 +5374,8 @@ def rotate(self, a, b): def same_quantum(self, a, b): """Returns True if the two operands have the same exponent. - The result is never affected by either the sign or the coefficient of - either operand. + The result is never affected by either the sign or the coefficient + of either operand. >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) False @@ -5442,8 +5447,8 @@ def shift(self, a, b): def sqrt(self, a): """Square root of a non-negative number to context precision. - If the result must be inexact, it is rounded using the round-half-even - algorithm. + If the result must be inexact, it is rounded using the + round-half-even algorithm. >>> ExtendedContext.sqrt(Decimal('0')) Decimal('0') diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 3306c8a274760be..4ba9b4070dff93e 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -10,7 +10,7 @@ import sys # Import _thread instead of threading to reduce startup cost from _thread import allocate_lock as Lock -if sys.platform in {'win32', 'cygwin'}: +if sys.platform == 'win32': from msvcrt import setmode as _setmode else: _setmode = None @@ -83,27 +83,28 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) - mode is an optional string that specifies the mode in which the file is - opened. It defaults to 'r' which means open for reading in text mode. Other - common values are 'w' for writing (truncating the file if it already - exists), 'x' for exclusive creation of a new file, and 'a' for appending - (which on some Unix systems, means that all writes append to the end of the - file regardless of the current seek position). In text mode, if encoding is - not specified the encoding used is platform dependent. (For reading and - writing raw bytes use binary mode and leave encoding unspecified.) The - available modes are: - - ========= =============================================================== + mode is an optional string that specifies the mode in which the file + is opened. It defaults to 'r' which means open for reading in text + mode. Other common values are 'w' for writing (truncating the file if + it already exists), 'x' for exclusive creation of a new file, and + 'a' for appending (which on some Unix systems, means that all writes + append to the end of the file regardless of the current seek position). + In text mode, if encoding is not specified the encoding used is platform + dependent. (For reading and writing raw bytes use binary mode and leave + encoding unspecified.) The available modes are: + + ========= ========================================================== Character Meaning - --------- --------------------------------------------------------------- + --------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing - 'a' open for writing, appending to the end of the file if it exists + 'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - ========= =============================================================== + ========= ========================================================== The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while @@ -111,23 +112,23 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, - even when the underlying operating system doesn't. Files opened in + even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as - bytes objects without any decoding. In text mode (the default, or when + bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. - Pass 0 to switch buffering off (only allowed in binary mode), 1 to select - line buffering (only usable in text mode), and an integer > 1 to indicate - the size of a fixed-size chunk buffer. When no buffering argument is - given, the default buffering policy works as follows: + Pass 0 to switch buffering off (only allowed in binary mode), 1 to + select line buffering (only usable in text mode), and an integer > 1 to + indicate the size of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy works as follows: - * Binary files are buffered in fixed-size chunks; the size of the buffer - is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) - when the device block size is available. - On most systems, the buffer will typically be 128 kilobytes long. + * Binary files are buffered in fixed-size chunks; the size of the buffer + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -147,8 +148,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, encoding error strings. newline is a string controlling how universal newlines works (it only - applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works - as follows: + applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It + works as follows: * On input, if newline is None, universal newlines mode is enabled. Lines in the input can end in '\n', '\r', or '\r\n', and @@ -164,17 +165,17 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, other legal values, any '\n' characters written are translated to the given string. - closedfd is a bool. If closefd is False, the underlying file descriptor will - be kept open when the file is closed. This does not work when a file name is - given and must be True in that case. + closedfd is a bool. If closefd is False, the underlying file descriptor + will be kept open when the file is closed. This does not work when + a file name is given and must be True in that case. The newly created file is non-inheritable. A custom opener can be used by passing a callable as *opener*. The - underlying file descriptor for the file object is then obtained by calling - *opener* with (*file*, *flags*). *opener* must return an open file - descriptor (passing os.open as *opener* results in functionality similar to - passing None). + underlying file descriptor for the file object is then obtained by + calling *opener* with (*file*, *flags*). *opener* must return an open + file descriptor (passing os.open as *opener* results in functionality + similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing @@ -351,10 +352,12 @@ def seek(self, pos, whence=0): interpreted relative to the position indicated by whence. Values for whence are ints: - * 0 -- start of stream (the default); offset should be zero or positive + * 0 -- start of stream (the default); offset should be zero or + positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset is usually negative - Some operating systems / file systems could provide additional values. + Some operating systems / file systems could provide additional + values. Return an int indicating the new absolute position. """ @@ -367,8 +370,8 @@ def tell(self): def truncate(self, pos=None): """Truncate file to size bytes. - Size defaults to the current IO position as reported by tell(). Return - the new size. + Size defaults to the current IO position as reported by tell(). + Return the new size. """ self._unsupported("truncate") @@ -492,7 +495,8 @@ def __exit__(self, *args): def fileno(self): """Returns underlying file descriptor (an int) if one exists. - An OSError is raised if the IO object does not use a file descriptor. + An OSError is raised if the IO object does not use a file + descriptor. """ self._unsupported("fileno") @@ -941,7 +945,7 @@ def read(self, size=-1): newpos = min(len(self._buffer), self._pos + size) b = self._buffer[self._pos : newpos] self._pos = newpos - return bytes(b) + return b.take_bytes() def read1(self, size=-1): """This is the same as read. @@ -1505,17 +1509,22 @@ class FileIO(RawIOBase): _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): - """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, - writing, exclusive creation or appending. The file will be created if it - doesn't exist when opened for writing or appending; it will be truncated - when opened for writing. A FileExistsError will be raised if it already - exists when opened for creating. Opening a file for creating implies - writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode - to allow simultaneous reading and writing. A custom opener can be used by - passing a callable as *opener*. The underlying file descriptor for the file - object is then obtained by calling opener with (*name*, *flags*). - *opener* must return an open file descriptor (passing os.open as *opener* - results in functionality similar to passing None). + """Open a file. + + The mode can be 'r' (default), 'w', 'x' or 'a' for reading, + writing, exclusive creation or appending. The file will be created + if it doesn't exist when opened for writing or appending; it will be + truncated when opened for writing. A FileExistsError will be raised + if it already exists when opened for creating. Opening a file for + creating implies writing so this mode behaves in a similar way to + 'w'. Add a '+' to the mode to allow simultaneous reading and + writing. + + A custom opener can be used by passing a callable as *opener*. + The underlying file descriptor for the file object is then obtained + by calling opener with (*name*, *flags*). *opener* must return + an open file descriptor (passing os.open as *opener* results in + functionality similar to passing None). """ if self._fd >= 0: # Have to close the existing file first. @@ -1754,8 +1763,8 @@ def write(self, b): """Write bytes b to file, return number written. Only makes one system call, so not all of the data may be written. - The number of bytes actually written is returned. In non-blocking mode, - returns None if the write would block. + The number of bytes actually written is returned. In non-blocking + mode, returns None if the write would block. """ self._checkClosed() self._checkWritable() @@ -1767,11 +1776,12 @@ def write(self, b): def seek(self, pos, whence=SEEK_SET): """Move to new file position. - Argument offset is a byte count. Optional argument whence defaults to - SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values - are SEEK_CUR or 1 (move relative to current position, positive or negative), - and SEEK_END or 2 (move relative to end of file, usually negative, although - many platforms allow seeking beyond the end of a file). + Argument offset is a byte count. Optional argument whence defaults + to SEEK_SET or 0 (offset from start of file, offset should be >= 0); + other values are SEEK_CUR or 1 (move relative to current position, + positive or negative), and SEEK_END or 2 (move relative to end of + file, usually negative, although many platforms allow seeking beyond + the end of a file). Note that not all file objects are seekable. """ @@ -1804,8 +1814,8 @@ def truncate(self, size=None): def close(self): """Close the file. - A closed file cannot be used for further I/O operations. close() may be - called more than once without error. + A closed file cannot be used for further I/O operations. + close() may be called more than once without error. """ if not self.closed: self._stat_atopen = None @@ -1903,8 +1913,8 @@ class TextIOBase(IOBase): def read(self, size=-1): """Read at most size characters from stream, where size is an int. - Read from underlying buffer until we have size characters or we hit EOF. - If size is negative or omitted, read until EOF. + Read from underlying buffer until we have size characters or we hit + EOF. If size is negative or omitted, read until EOF. Returns a string. """ @@ -1918,7 +1928,7 @@ def truncate(self, pos=None): """Truncate size to pos, where pos is an int.""" self._unsupported("truncate") - def readline(self): + def readline(self, size=-1, /): """Read until newline or EOF. Returns an empty string if EOF is hit immediately. diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index a22b0297b24ea07..17bf5cdc819542d 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from itertools import chain from tokenize import TokenInfo +from .fancycompleter import safe_getattr TYPE_CHECKING = False @@ -71,41 +72,69 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self._curr_sys_path: list[str] = sys.path[:] self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> tuple[list[str], CompletionAction | None] | None: + def get_completions( + self, line: str, *, include_values: bool = True + ) -> tuple[list[str], list[Any], CompletionAction | None] | None: """Return the next possible import completions for 'line'. For attributes completion, if the module to complete from is not imported, also return an action (prompt + callback to run if the user press TAB again) to import the module. + + If *include_values* is false, the returned values list is empty and + attribute values are not resolved. """ result = ImportParser(line).parse() if not result: return None try: - return self.complete(*result) + return self.complete(*result, include_values=include_values) except Exception: # Some unexpected error occurred, make it look like # no completions are available - return [], None - - def complete(self, from_name: str | None, name: str | None) -> tuple[list[str], CompletionAction | None]: + return [], [], None + + def complete( + self, + from_name: str | None, + name: str | None, + *, + include_values: bool = True, + ) -> tuple[list[str], list[Any], CompletionAction | None]: if from_name is None: # import x.y.z assert name is not None path, prefix = self.get_path_and_prefix(name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules], None + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None if name is None: # from x.y.z path, prefix = self.get_path_and_prefix(from_name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules], None + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None # from x.y import z submodules = self.find_modules(from_name, name) - attributes, action = self.find_attributes(from_name, name) - return sorted({*submodules, *attributes}), action + attr_names, attr_module, action = self._find_attributes(from_name, name) + all_names = sorted({*submodules, *attr_names}) + if not include_values: + return all_names, [], action + + # Build values list matching the sorted order: + # submodules use `sys` as a dummy value so they get the 'module' color, + # attributes use their actual value. + attr_map = {} + if attr_module is not None: + attr_map = {n: safe_getattr(attr_module, n) for n in attr_names} + all_values = [attr_map[n] if n in attr_map else sys for n in all_names] + return all_names, all_values, action def find_modules(self, path: str, prefix: str) -> list[str]: """Find all modules under 'path' that start with 'prefix'.""" @@ -166,31 +195,43 @@ def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: return (isinstance(module_info.module_finder, FileFinder) and module_info.module_finder.path == self._stdlib_path) - def find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + def find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], list[Any], CompletionAction | None]: """Find all attributes of module 'path' that start with 'prefix'.""" - attributes, action = self._find_attributes(path, prefix) - # Filter out invalid attribute names - # (for example those containing dashes that cannot be imported with 'import') - return [attr for attr in attributes if attr.isidentifier()], action + attributes, module, action = self._find_attributes(path, prefix) + if module is not None: + values = [safe_getattr(module, attr) for attr in attributes] + else: + values = [] + return attributes, values, action - def _find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + def _find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], ModuleType | None, CompletionAction | None]: path = self._resolve_relative_path(path) # type: ignore[assignment] if path is None: - return [], None + return [], None, None imported_module = sys.modules.get(path) if not imported_module: if path in self._failed_imports: # Do not propose to import again - return [], None + return [], None, None imported_module = self._maybe_import_module(path) if not imported_module: - return [], self._get_import_completion_action(path) + return [], None, self._get_import_completion_action(path) try: module_attributes = dir(imported_module) except Exception: module_attributes = [] - return [attr_name for attr_name in module_attributes - if self.is_suggestion_match(attr_name, prefix)], None + # Filter out invalid attribute names, such as dashes that cannot be + # imported with 'import'. + names = [ + attr_name for attr_name in module_attributes + if (self.is_suggestion_match(attr_name, prefix) + and attr_name.isidentifier()) + ] + return names, imported_module, None def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index f783e8db36b0287..7eb3ebac84124b5 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -32,7 +32,7 @@ # types Command = commands.Command if TYPE_CHECKING: - from .types import CommandName, CompletionAction, Keymap, KeySpec + from .types import CompletionAction, Keymap def prefix(wordlist: list[str], j: int = 0) -> str: diff --git a/Lib/_pyrepl/fancycompleter.py b/Lib/_pyrepl/fancycompleter.py index 7a639afd74ef3cc..ac4f0afdbc721c3 100644 --- a/Lib/_pyrepl/fancycompleter.py +++ b/Lib/_pyrepl/fancycompleter.py @@ -3,11 +3,53 @@ # # All Rights Reserved """Colorful tab completion for Python prompt""" +from __future__ import annotations + from _colorize import ANSIColors, get_colors, get_theme import rlcompleter import keyword import types +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import Any + from _colorize import Theme + + +def safe_getattr(obj, name): + # Mirror rlcompleter's safeguards so completion does not + # call properties or reify lazy module attributes. + if isinstance(getattr(type(obj), name, None), property): + return None + if (isinstance(obj, types.ModuleType) + and isinstance(obj.__dict__.get(name), types.LazyImportType) + ): + return obj.__dict__.get(name) + return getattr(obj, name, None) + + +def colorize_matches(names: list[str], values: list[Any], theme: Theme) -> list[str]: + return [ + _color_for_obj(name, obj, theme) + for name, obj in zip(names, values) + ] + +def _color_for_obj(name: str, value: Any, theme: Theme) -> str: + t = type(value) + color = _color_by_type(t, theme) + return f"{color}{name}{ANSIColors.RESET}" + + +def _color_by_type(t, theme): + typename = t.__name__ + # this is needed e.g. to turn method-wrapper into method_wrapper, + # because if we want _colorize.FancyCompleter to be "dataclassable" + # our keys need to be valid identifiers. + typename = typename.replace('-', '_').replace('.', '_') + return getattr(theme.fancycompleter, typename, ANSIColors.RESET) + + class Completer(rlcompleter.Completer): """ When doing something like a.b., keep the full a.b.attr completion @@ -143,21 +185,7 @@ def _attr_matches(self, text): word[:n] == attr and not (noprefix and word[:n+1] == noprefix) ): - # Mirror rlcompleter's safeguards so completion does not - # call properties or reify lazy module attributes. - if isinstance(getattr(type(thisobject), word, None), property): - value = None - elif ( - isinstance(thisobject, types.ModuleType) - and isinstance( - thisobject.__dict__.get(word), - types.LazyImportType, - ) - ): - value = thisobject.__dict__.get(word) - else: - value = getattr(thisobject, word, None) - + value = safe_getattr(thisobject, word) names.append(word) values.append(value) if names or not noprefix: @@ -170,23 +198,7 @@ def _attr_matches(self, text): return expr, attr, names, values def colorize_matches(self, names, values): - return [ - self._color_for_obj(name, obj) - for name, obj in zip(names, values) - ] - - def _color_for_obj(self, name, value): - t = type(value) - color = self._color_by_type(t) - return f"{color}{name}{ANSIColors.RESET}" - - def _color_by_type(self, t): - typename = t.__name__ - # this is needed e.g. to turn method-wrapper into method_wrapper, - # because if we want _colorize.FancyCompleter to be "dataclassable" - # our keys need to be valid identifiers. - typename = typename.replace('-', '_').replace('.', '_') - return getattr(self.theme.fancycompleter, typename, ANSIColors.RESET) + return colorize_matches(names, values, self.theme) def commonprefix(names): diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 7e4dd801c84d5c2..832e67b534f2969 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -38,7 +38,7 @@ ) from .layout import LayoutMap, LayoutResult, LayoutRow, WrappedRow, layout_content_lines from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay -from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, wlen, gen_colors +from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, gen_colors from .trace import trace diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index f8f1727d2a1d1f4..e4370b0d1462eaf 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -40,7 +40,7 @@ from .completing_reader import CompletingReader, stripcolor from .console import Console as ConsoleType from ._module_completer import ModuleCompleter, make_default_module_completer -from .fancycompleter import Completer as FancyCompleter +from .fancycompleter import Completer as FancyCompleter, colorize_matches Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] @@ -104,6 +104,7 @@ class ReadlineConfig: readline_completer: Completer | None = None completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") module_completer: ModuleCompleter = field(default_factory=make_default_module_completer) + colorize_completions: Callable[[list[str], list[Any]], list[str]] | None = None @dataclass(kw_only=True) class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): @@ -169,8 +170,17 @@ def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None return result, None def get_module_completions(self) -> tuple[list[str], CompletionAction | None] | None: - line = self.get_line() - return self.config.module_completer.get_completions(line) + line = stripcolor(self.get_line()) + colorize_completions = self.config.colorize_completions + result = self.config.module_completer.get_completions( + line, include_values=bool(colorize_completions) + ) + if result is None: + return None + names, values, action = result + if colorize_completions: + names = colorize_completions(names, values) + return names, action def get_trimmed_history(self, maxlength: int) -> list[str]: if maxlength >= 0: @@ -616,13 +626,19 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) - _wrapper.config.module_completer = ModuleCompleter(namespace) use_basic_completer = ( not sys.flags.ignore_environment and os.getenv("PYTHON_BASIC_COMPLETER") ) completer_cls = RLCompleter if use_basic_completer else FancyCompleter - _wrapper.config.readline_completer = completer_cls(namespace).complete + completer = completer_cls(namespace) + _wrapper.config.readline_completer = completer.complete + if isinstance(completer, FancyCompleter) and completer.use_colors: + theme = completer.theme + def _colorize(names: list[str], values: list[object]) -> list[str]: + return colorize_matches(names, values, theme) + _wrapper.config.colorize_completions = _colorize + _wrapper.config.module_completer = ModuleCompleter(namespace) # this is not really what readline.c does. Better than nothing I guess import builtins diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 9c4644db53e3436..9a4f8e0c623abfa 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -34,7 +34,7 @@ from collections.abc import Callable from dataclasses import dataclass from fcntl import ioctl -from typing import TYPE_CHECKING, cast, overload +from typing import TYPE_CHECKING, overload from . import terminfo from .console import Console, Event diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index b50426c31ead53a..230dae35af665ab 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -259,7 +259,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), TI(string="match"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{" | "~" | "...") ): return True case ( @@ -274,7 +274,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "-" | "[" | "{") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{") ): return True case ( diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index 84551e3546eb6ee..5259a36123ea276 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -55,7 +55,6 @@ def __setup(self): if not data: data = self.__data self.__lines = data.split('\n') - self.__linecnt = len(self.__lines) def __repr__(self): self.__setup() diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 746b0907c1d9f4e..59ac96745aa15e2 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -238,7 +238,7 @@ def __calc_date_time(self): current_format = current_format.replace(tz, "%Z") # Transform all non-ASCII digits to digits in range U+0660 to U+0669. if not current_format.isascii() and self.LC_alt_digits is None: - current_format = re_sub(r'\d(?', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None): @@ -142,6 +141,8 @@ def dump( If show_empty is False, then empty lists and fields that are None will be omitted from the output for better readability. """ + from _colorize import get_theme + t = get_theme(force_color=color, force_no_color=not color).ast def _format(node, level=0): @@ -629,9 +630,11 @@ def __new__(cls, dims=(), **kwargs): def _dims_getter(self): """Deprecated. Use elts instead.""" + warnings._deprecated(f"ast.Tuple.dims", remove=(3, 21)) return self.elts def _dims_setter(self, value): + warnings._deprecated(f"ast.Tuple.dims", remove=(3, 21)) self.elts = value Tuple.dims = property(_dims_getter, _dims_setter) @@ -665,7 +668,7 @@ def main(args=None): parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('infile', nargs='?', default='-', - help='the file to parse; defaults to stdin') + help='the file to parse; defaults to `stdin`') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), help='specify what kind of code must be parsed') @@ -678,8 +681,8 @@ def main(args=None): help='indentation of nodes (number of spaces)') parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', - help='Python version in the format 3.x ' - '(for example, 3.10)') + help='Python version in the format `3.x` ' + '(for example, `3.10`)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', help='optimization level for parser') @@ -708,9 +711,28 @@ def main(args=None): tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) + from _colorize import can_colorize print(dump(tree, include_attributes=args.include_attributes, color=can_colorize(file=sys.stdout), indent=args.indent, show_empty=args.show_empty)) +_deprecated = { + 'slice': globals().pop("slice"), + 'Index': globals().pop("Index"), + 'ExtSlice': globals().pop("ExtSlice"), + 'Suite': globals().pop("Suite"), + 'AugLoad': globals().pop("AugLoad"), + 'AugStore': globals().pop("AugStore"), + 'Param': globals().pop("Param") +} + +def __getattr__(attr): + try: + val = _deprecated[attr] + except KeyError: + raise AttributeError(f"module 'ast' has no attribute {attr!r}") from None + warnings._deprecated(f"ast.{attr}", remove=(3, 21)) + return val + if __name__ == '__main__': main() diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 37eba9657ac5a88..7f0565d0b8ddc76 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -159,7 +159,6 @@ def interrupt(self) -> None: parser = argparse.ArgumentParser( prog="python3 -m asyncio", description="Interactive asyncio shell and CLI tools", - color=True, ) subparsers = parser.add_subparsers(help="sub-commands", dest="command") ps = subparsers.add_parser( diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 7a6837546d930f3..e6c72e3d5b5487e 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -31,6 +31,7 @@ import traceback import warnings import weakref +import inspect try: import ssl @@ -487,10 +488,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro, **kwargs)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object, and **kwargs will be - arbitrary keyword arguments that should be passed on to Task. - The callable must return a Task. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the + active event loop, 'coro' will be a coroutine object, and **kwargs + will be arbitrary keyword arguments that should be passed on to + Task. The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -726,8 +727,8 @@ def run_until_complete(self, future): def stop(self): """Stop running the event loop. - Every callback already scheduled will still run. This simply informs - run_forever to stop looping after a complete iteration. + Every callback already scheduled will still run. This simply + informs run_forever to stop looping after a complete iteration. """ self._stopping = True @@ -840,7 +841,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines._iscoroutinefunction(callback)): + inspect.iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -968,7 +969,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): f"and file {file!r} combination") async def _sock_sendfile_fallback(self, sock, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = ( min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) @@ -1075,12 +1076,12 @@ async def create_connection( Create a streaming transport connection to a given internet host and port: socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_STREAM. protocol_factory must be - a callable returning a protocol instance. + family if specified), socket type SOCK_STREAM. protocol_factory must + be a callable returning a protocol instance. - This method is a coroutine which will try to establish the connection - in the background. When successful, the coroutine returns a - (transport, protocol) pair. + This method is a coroutine which will try to establish the + connection in the background. When successful, the coroutine + returns a (transport, protocol) pair. """ if server_hostname is not None and not ssl: raise ValueError('server_hostname is only meaningful with ssl') @@ -1285,7 +1286,6 @@ async def sendfile(self, transport, file, offset=0, count=None, raise RuntimeError( f"fallback is disabled and native sendfile is not " f"supported for transport {transport!r}") - return await self._sendfile_fallback(transport, file, offset, count) @@ -1294,7 +1294,7 @@ async def _sendfile_native(self, transp, file, offset, count): "sendfile syscall is not supported") async def _sendfile_fallback(self, transp, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = min(count, 16384) if count else 16384 buf = bytearray(blocksize) @@ -1550,11 +1550,11 @@ async def create_server( The host parameter can be a string, in that case the TCP server is bound to host and port. - The host parameter can also be a sequence of strings and in that case - the TCP server is bound to all hosts of the sequence. If a host - appears multiple times (possibly indirectly e.g. when hostnames - resolve to the same IP address), the server is only bound once to that - host. + The host parameter can also be a sequence of strings and in that + case the TCP server is bound to all hosts of the sequence. If + a host appears multiple times (possibly indirectly e.g. when + hostnames resolve to the same IP address), the server is only bound + once to that host. Return a Server object which can be used to stop the service. diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index a51319cb72a6a9b..3a4246f6ccd4c24 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -1,7 +1,6 @@ -__all__ = 'iscoroutinefunction', 'iscoroutine' +__all__ = ('iscoroutine',) import collections.abc -import inspect import os import sys import types @@ -13,25 +12,6 @@ def _is_debug_mode(): bool(os.environ.get('PYTHONASYNCIODEBUG'))) -# A marker for iscoroutinefunction. -_is_coroutine = object() - - -def iscoroutinefunction(func): - import warnings - """Return True if func is a decorated coroutine function.""" - warnings._deprecated("asyncio.iscoroutinefunction", - f"{warnings._DEPRECATED_MSG}; " - "use inspect.iscoroutinefunction() instead", - remove=(3,16)) - return _iscoroutinefunction(func) - - -def _iscoroutinefunction(func): - return (inspect.iscoroutinefunction(func) or - getattr(func, '_is_coroutine', None) is _is_coroutine) - - # Prioritize native coroutine check to speed-up # asyncio.iscoroutine. _COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index a7fb55982abe9ca..42d8408a38dfd1f 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -374,8 +374,8 @@ async def create_server( If host is an empty string or None all interfaces are assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be - a sequence (e.g. list) of hosts to bind to. + one for IPv4 and another one for IPv6). The host parameter can also + be a sequence (e.g. list) of hosts to bind to. family can be set to either AF_INET or AF_INET6 to force the socket to use IPv4 or IPv6. If not set it will be determined @@ -415,8 +415,9 @@ async def create_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -479,8 +480,9 @@ async def create_unix_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -511,8 +513,8 @@ async def create_datagram_endpoint(self, protocol_factory, protocol_factory must be a callable returning a protocol instance. - socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on - host (or family if specified), socket type SOCK_DGRAM. + socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending + on host (or family if specified), socket type SOCK_DGRAM. reuse_address tells the kernel to reuse a local socket in TIME_WAIT state, without waiting for its natural timeout to @@ -552,7 +554,8 @@ async def connect_read_pipe(self, protocol_factory, pipe): async def connect_write_pipe(self, protocol_factory, pipe): """Register write pipe in event loop. - protocol_factory should instantiate object with BaseProtocol interface. + protocol_factory should instantiate object with BaseProtocol + interface. Pipe is file-like object already switched to nonblocking. Return pair (transport, protocol), where transport support WriteTransport interface.""" diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index b5bfeb1630a1590..35f7fa62140bd49 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -112,13 +112,13 @@ def capture_call_graph( optional keyword-only 'depth' argument can be used to skip the specified number of frames from top of the stack. - If the optional keyword-only 'limit' argument is provided, each call stack - in the resulting graph is truncated to include at most ``abs(limit)`` - entries. If 'limit' is positive, the entries left are the closest to - the invocation point. If 'limit' is negative, the topmost entries are - left. If 'limit' is omitted or None, all entries are present. - If 'limit' is 0, the call stack is not captured at all, only - "awaited by" information is present. + If the optional keyword-only 'limit' argument is provided, each call + stack in the resulting graph is truncated to include at most + ``abs(limit)`` entries. If 'limit' is positive, the entries left are + the closest to the invocation point. If 'limit' is negative, the + topmost entries are left. If 'limit' is omitted or None, all entries + are present. If 'limit' is 0, the call stack is not captured at all, + only "awaited by" information is present. """ loop = events._get_running_loop() diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index fa3a94764b507ae..c59ea15111bef2b 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -145,10 +145,7 @@ def _wake_up_first(self): """Ensure that the first waiter will wake up.""" if not self._waiters: return - try: - fut = next(iter(self._waiters)) - except StopIteration: - return + fut = next(iter(self._waiters)) # .done() means that the waiter is already set to wake up. if not fut.done(): @@ -158,10 +155,10 @@ def _wake_up_first(self): class Event(mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Event. - Class implementing event objects. An event manages a flag that can be set - to true with the set() method and reset to false with the clear() method. - The wait() method blocks until the flag is true. The flag is initially - false. + Class implementing event objects. An event manages a flag that can be + set to true with the set() method and reset to false with the clear() + method. The wait() method blocks until the flag is true. The flag is + initially false. """ def __init__(self): @@ -353,9 +350,9 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): """A Semaphore implementation. A semaphore manages an internal counter which is decremented by each - acquire() call and incremented by each release() call. The counter - can never go below zero; when acquire() finds that it is zero, it blocks, - waiting until some other thread calls release(). + acquire() call and incremented by each release() call. The counter + can never go below zero; when acquire() finds that it is zero, it + blocks, waiting until some other thread calls release(). Semaphores also support the context management protocol. @@ -511,8 +508,8 @@ async def __aexit__(self, *args): async def wait(self): """Wait for the barrier. - When the specified number of tasks have started waiting, they are all - simultaneously awoken. + When the specified number of tasks have started waiting, they are + all simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. """ async with self._cond: diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 2dc1569d7807911..cf2902b4c76559e 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -756,8 +756,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): offset += blocksize total_sent += blocksize finally: - if total_sent > 0: - file.seek(offset) + file.seek(offset) async def _sendfile_native(self, transp, file, offset, count): resume_reading = transp.is_reading() diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 756216fac809329..30004f2bc9bacd2 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -33,9 +33,9 @@ class QueueShutDown(Exception): class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. - If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "await put()" will block when the - queue reaches maxsize, until an item is removed by get(). + If maxsize is less than or equal to zero, the queue size is infinite. + If it is an integer greater than 0, then "await put()" will block when + the queue reaches maxsize, until an item is removed by get(). Unlike queue.Queue, you can reliably know this Queue's size with qsize(), since your single-threaded asyncio application won't be @@ -174,8 +174,8 @@ async def get(self): If queue is empty, wait until an item is available. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ while self.empty(): if self._is_shutdown and self.empty(): @@ -203,10 +203,11 @@ async def get(self): def get_nowait(self): """Remove and return an item from the queue. - Return an item if one is immediately available, else raise QueueEmpty. + Return an item if one is immediately available, else raise + QueueEmpty. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ if self.empty(): if self._is_shutdown: @@ -223,12 +224,12 @@ def task_done(self): a subsequent call to task_done() tells the queue that the processing on the task is complete. - If a join() is currently blocking, it will resume when all items have - been processed (meaning that a task_done() call was received for every - item that had been put() into the queue). + If a join() is currently blocking, it will resume when all items + have been processed (meaning that a task_done() call was received + for every item that had been put() into the queue). - Raises ValueError if called more times than there were items placed in - the queue. + Raises ValueError if called more times than there were items placed + in the queue. """ if self._unfinished_tasks <= 0: raise ValueError('task_done() called too many times') @@ -239,10 +240,11 @@ def task_done(self): async def join(self): """Block until all items in the queue have been gotten and processed. - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls task_done() to - indicate that the item was retrieved and all work on it is complete. - When the count of unfinished tasks drops to zero, join() unblocks. + The count of unfinished tasks goes up whenever an item is added to + the queue. The count goes down whenever a consumer calls + task_done() to indicate that the item was retrieved and all work on + it is complete. When the count of unfinished tasks drops to zero, + join() unblocks. """ if self._unfinished_tasks > 0: await self._finished.wait() diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index ba37e003a655c0a..774a317564df22c 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -35,7 +35,8 @@ class Runner: with asyncio.Runner(debug=True) as runner: runner.run(main()) - The run() method can be called multiple times within the runner's context. + The run() method can be called multiple times within the runner's + context. This can be useful for interactive console (e.g. IPython), unittest runners, console tools, -- everywhere when async code diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 961dbfb4b96303e..4fa4dd5a885236e 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -53,7 +53,7 @@ def _test_selector_event(selector, fd, event): class BaseSelectorEventLoop(base_events.BaseEventLoop): """Selector event loop. - See events.EventLoop for API specification. + See events.AbstractEventLoop for API specification. """ def __init__(self, selector=None): @@ -533,11 +533,12 @@ def _sock_recvfrom_into(self, fut, sock, buf, bufsize): async def sock_sendall(self, sock, data): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -586,11 +587,12 @@ def _sock_sendall(self, fut, sock, view, pos): async def sock_sendto(self, sock, data, address): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -701,10 +703,11 @@ def _sock_connect_cb(self, fut, sock, address): async def sock_accept(self, sock): """Accept a connection. - The socket must be bound to an address and listening for connections. - The return value is a pair (conn, address) where conn is a new socket - object usable to send and receive data on the connection, and address - is the address bound to the socket on the other end of the connection. + The socket must be bound to an address and listening for + connections. The return value is a pair (conn, address) where + conn is a new socket object usable to send and receive data on the + connection, and address is the address bound to the socket on the + other end of the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -1233,7 +1236,6 @@ def close(self): class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): - _buffer_factory = collections.deque _header_size = 8 def __init__(self, loop, sock, protocol, address=None, diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index d2db1a930c2ad2b..a28c11e928f806a 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -539,17 +539,17 @@ async def _wait_for_data(self, func_name): self._waiter = None async def readline(self): - """Read chunk of data from the stream until newline (b'\n') is found. + r"""Read chunk of data from the stream until newline (b'\n') is found. - On success, return chunk that ends with newline. If only partial + On success, return chunk that ends with newline. If only partial line can be read due to EOF, return incomplete line without - terminating newline. When EOF was reached while no bytes read, empty - bytes object is returned. + terminating newline. When EOF was reached while no bytes read, + empty bytes object is returned. - If limit is reached, ValueError will be raised. In that case, if + If limit is reached, ValueError will be raised. In that case, if newline was found, complete line including newline will be removed - from internal buffer. Else, internal buffer will be cleared. Limit is - compared against part of the line without newline. + from internal buffer. Else, internal buffer will be cleared. + Limit is compared against part of the line without newline. If stream was paused, this function will automatically resume it if needed. diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 45dfebc65904fce..e1ec025791a52e6 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -289,14 +289,14 @@ def cancel(self): """Cancel the task group `cancel()` will be called on any tasks in the group that aren't yet - done, as well as the parent (body) of the group. This will cause the - task group context manager to exit *without* `asyncio.CancelledError` - being raised. - - If `cancel()` is called before entering the task group, the group will be - cancelled upon entry. This is useful for patterns where one piece of - code passes an unused TaskGroup instance to another in order to have - the ability to cancel anything run within the group. + done, as well as the parent (body) of the group. This will cause + the task group context manager to exit *without* + `asyncio.CancelledError` being raised. + + If `cancel()` is called before entering the task group, the group + will be cancelled upon entry. This is useful for patterns where + one piece of code passes an unused TaskGroup instance to another in + order to have the ability to cancel anything run within the group. `cancel()` is idempotent and may be called after the task group has already exited. diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index fbd5c39a7c56ac4..14d3c1ceb58cacb 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -624,10 +624,11 @@ def as_completed(fs, *, timeout=None): Run the supplied awaitables concurrently. The returned object can be iterated to obtain the results of the awaitables as they finish. - The object returned can be iterated as an asynchronous iterator or a plain - iterator. When asynchronous iteration is used, the originally-supplied - awaitables are yielded if they are tasks or futures. This makes it easy to - correlate previously-scheduled tasks with their results: + The object returned can be iterated as an asynchronous iterator or + a plain iterator. When asynchronous iteration is used, the + originally-supplied awaitables are yielded if they are tasks or + futures. This makes it easy to correlate previously-scheduled tasks + with their results: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) @@ -643,26 +644,27 @@ def as_completed(fs, *, timeout=None): else: print("IPv4 connection established.") - During asynchronous iteration, implicitly-created tasks will be yielded for - supplied awaitables that aren't tasks or futures. + During asynchronous iteration, implicitly-created tasks will be + yielded for supplied awaitables that aren't tasks or futures. - When used as a plain iterator, each iteration yields a new coroutine that - returns the result or raises the exception of the next completed awaitable. - This pattern is compatible with Python versions older than 3.13: + When used as a plain iterator, each iteration yields a new coroutine + that returns the result or raises the exception of the next completed + awaitable. This pattern is compatible with Python versions older than + 3.13: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) tasks = [ipv4_connect, ipv6_connect] for next_connect in as_completed(tasks): - # next_connect is not one of the original task objects. It must be - # awaited to obtain the result value or raise the exception of the - # awaitable that finishes next. + # next_connect is not one of the original task objects. It must + # be awaited to obtain the result value or raise the exception + # of the awaitable that finishes next. reader, writer = await next_connect - A TimeoutError is raised if the timeout occurs before all awaitables are - done. This is raised by the async for loop during asynchronous iteration or - by the coroutines yielded during plain iteration. + A TimeoutError is raised if the timeout occurs before all awaitables + are done. This is raised by the async for loop during asynchronous + iteration or by the coroutines yielded during plain iteration. """ if inspect.isawaitable(fs): raise TypeError( @@ -1030,21 +1032,22 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any + callable that returns a Task-compatible object and has a signature + compatible with `Task.__init__`; it must have the `eager_start` + keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in - this case there's no need to call `create_eager_task_factory()` - directly. Instead the global `eager_task_factory` instance can be - used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ def factory(loop, coro, *, eager_start=True, **kwargs): return custom_task_constructor( diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py index db048a8231de166..5001351b0976ecd 100644 --- a/Lib/asyncio/threads.py +++ b/Lib/asyncio/threads.py @@ -17,7 +17,8 @@ async def to_thread(func, /, *args, **kwargs): allowing context variables from the main thread to be accessed in the separate thread. - Return a coroutine that can be awaited to get the eventual result of *func*. + Return a coroutine that can be awaited to get the eventual result of + *func*. """ loop = events.get_running_loop() ctx = contextvars.copy_context() diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 09342dc7c1310b0..65ddc285abd971d 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -25,7 +25,8 @@ class _State(enum.Enum): class Timeout: """Asynchronous context manager for cancelling overdue coroutines. - Use `timeout()` or `timeout_at()` rather than instantiating this class directly. + Use `timeout()` or `timeout_at()` rather than instantiating this class + directly. """ def __init__(self, when: float | None) -> None: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 49e8067ee7b4e59..676e0b670faa49b 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -12,6 +12,7 @@ import sys import threading import warnings +import inspect from . import base_events from . import base_subprocess @@ -54,7 +55,8 @@ def waitstatus_to_exitcode(status): class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. - Adds signal handling and UNIX Domain Socket support to SelectorEventLoop. + Adds signal handling and UNIX Domain Socket support to + SelectorEventLoop. """ def __init__(self, selector=None): @@ -94,7 +96,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines._iscoroutinefunction(callback)): + inspect.iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -384,12 +386,12 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # order to simplify the common case. self.remove_writer(registered_fd) if fut.cancelled(): - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) return if count: blocksize = count - total_sent if blocksize <= 0: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) return @@ -423,20 +425,20 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # plain send(). err = exceptions.SendfileNotAvailableError( "os.sendfile call failed") - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(err) else: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) else: if sent == 0: # EOF - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) else: offset += sent @@ -447,9 +449,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fd, sock, fileno, offset, count, blocksize, total_sent) - def _sock_sendfile_update_filepos(self, fileno, offset, total_sent): - if total_sent > 0: - os.lseek(fileno, offset, os.SEEK_SET) + def _sock_sendfile_update_filepos(self, fileno, offset): + # After this helper runs, the source fd's lseek pointer is at offset." + os.lseek(fileno, offset, os.SEEK_SET) def _sock_add_cancellation_callback(self, fut, sock): def cb(fut): @@ -834,7 +836,8 @@ class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w = None - if stdin == subprocess.PIPE and sys.platform.startswith('aix'): + if (stdin == subprocess.PIPE + and (sys.platform.startswith('aix') or sys.platform == 'cygwin')): # Use a socket pair for stdin on AIX, since it does not # support selecting read events on the write end of a # socket (which we use in order to detect closing of the @@ -886,8 +889,8 @@ def _do_wait(self, pid, pidfd, callback, args): pid) else: returncode = waitstatus_to_exitcode(status) - - os.close(pidfd) + finally: + os.close(pidfd) callback(pid, returncode, *args) class _ThreadedChildWatcher: diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 5f75b17d8ca649b..0bf7732136f1f8e 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -610,6 +610,9 @@ def sendfile(self, sock, file, offset, count): ov = _overlapped.Overlapped(NULL) offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff + # TransmitFile ignores OVERLAPPED.Offset for handles not opened with + # FILE_FLAG_OVERLAPPED, so seek the CRT file pointer to match. + file.seek(offset) ov.TransmitFile(sock.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index acd49441131b042..d6393f0b1ffee5d 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -111,8 +111,9 @@ def fileno(self): def close(self, *, CloseHandle=_winapi.CloseHandle): if self._handle is not None: - CloseHandle(self._handle) + handle = self._handle self._handle = None + CloseHandle(handle) def __del__(self, _warn=warnings.warn): if self._handle is not None: diff --git a/Lib/base64.py b/Lib/base64.py index 4b810e08569e5ba..fa562f74a810345 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,4 +1,4 @@ -"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" +"""Base16, Base32, Base64 (RFC 4648), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support @@ -219,7 +219,7 @@ def urlsafe_b64decode(s, *, padded=False): characters present in the input. ''' _B32_DECODE_MAP01_DOCSTRING = ''' -RFC 3548 allows for optional mapping of the digit 0 (zero) to the +RFC 4648 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be @@ -266,7 +266,7 @@ def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', extra_args='') -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns +# RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. def b16encode(s, *, wrapcol=0): @@ -315,16 +315,20 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. + feature is not supported by the standard encoding used in PDF. If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - pad controls whether the input is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. + pad controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by btoa, + producing an exact multiple of 5 bytes of output. + + adobe controls whether the encoded byte sequence is framed with <~ + and ~>, as in a PostScript base-85 string literal. Note that + while ASCII85Decode streams in PDF documents must be terminated + with ~>, they must not use a leading <~. - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. """ return binascii.b2a_ascii85(b, foldspaces=foldspaces, adobe=adobe, wrapcol=wrapcol, pad=pad) @@ -333,12 +337,14 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. + foldspaces is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII + 0x20). This feature is not supported by the standard Ascii85 + encoding used in PDF and PostScript. - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). + adobe controls whether the <~ and ~> markers are present. While + the leading <~ is not required, the input must end with ~>, or a + ValueError is raised. ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default @@ -358,8 +364,10 @@ def b85encode(b, pad=False, *, wrapcol=0): If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of 4 + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes. """ return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) @@ -379,8 +387,10 @@ def z85encode(s, pad=False, *, wrapcol=0): If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. """ return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, alphabet=binascii.Z85_ALPHABET) diff --git a/Lib/calendar.py b/Lib/calendar.py index fa9775ab040b149..92fe6b7723fe268 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -922,7 +922,7 @@ def main(args=None): "-t", "--type", default="text", choices=("text", "html"), - help="output type (text or html)" + help="output type (`text` or `html`)" ) parser.add_argument( "-f", "--first-weekday", diff --git a/Lib/code.py b/Lib/code.py index f7e275d8801b7c3..df1d7199e339341 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -385,7 +385,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-q', action='store_true', help="don't print version and copyright messages") args = parser.parse_args() diff --git a/Lib/codecs.py b/Lib/codecs.py index e4a8010aba90a50..af6ab031157e790 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -93,7 +93,7 @@ class CodecInfo(tuple): def __new__(cls, encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None, - *, _is_text_encoding=None): + *, _is_text_encoding=None, _expat_decoding_table=None): self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter)) self.name = name self.encode = encode @@ -104,6 +104,8 @@ def __new__(cls, encode, decode, streamreader=None, streamwriter=None, self.streamreader = streamreader if _is_text_encoding is not None: self._is_text_encoding = _is_text_encoding + if _expat_decoding_table is not None: + self._expat_decoding_table = _expat_decoding_table return self def __repr__(self): diff --git a/Lib/compileall.py b/Lib/compileall.py index c452aed135838f9..812a496611e043f 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -326,7 +326,6 @@ def main(): parser = argparse.ArgumentParser( description='Utilities to support installing Python libraries.', - color=True, ) parser.add_argument('-l', action='store_const', const=0, default=None, dest='maxlevels', @@ -338,10 +337,10 @@ def main(): parser.add_argument('-f', action='store_true', dest='force', help='force rebuild even if timestamps are up to date') parser.add_argument('-q', action='count', dest='quiet', default=0, - help='output only error messages; -qq will suppress ' + help='output only error messages; `-qq` will suppress ' 'the error messages as well.') parser.add_argument('-b', action='store_true', dest='legacy', - help='use legacy (pre-PEP3147) compiled file locations') + help='use legacy (pre-PEP 3147) compiled file locations') parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, help=('directory to prepend to file paths for use in ' 'compile-time tracebacks and in runtime ' @@ -367,28 +366,28 @@ def main(): 'of each file considered for compilation')) parser.add_argument('-i', metavar='FILE', dest='flist', help=('add all the files and directories listed in ' - 'FILE to the list considered for compilation; ' - 'if "-", names are read from stdin')) + '`FILE` to the list considered for compilation; ' + 'if `"-"`, names are read from `stdin`')) parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' - 'to the equivalent of -l sys.path')) + 'to the equivalent of `-l` `sys.path`')) parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') invalidation_modes = [mode.name.lower().replace('_', '-') for mode in py_compile.PycInvalidationMode] parser.add_argument('--invalidation-mode', choices=sorted(invalidation_modes), - help=('set .pyc invalidation mode; defaults to ' - '"checked-hash" if the SOURCE_DATE_EPOCH ' + help=('set `.pyc` invalidation mode; defaults to ' + '`"checked-hash"` if the `SOURCE_DATE_EPOCH` ' 'environment variable is set, and ' - '"timestamp" otherwise.')) + '`"timestamp"` otherwise.')) parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses the optimization level ' - 'of the Python interpreter itself (see -O).')) + 'Default is `-1` which uses the optimization level ' + 'of the Python interpreter itself (see `-O`).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', - help='Ignore symlinks pointing outsite of the DIR') + help='Ignore symlinks pointing outsite of the `DIR`') parser.add_argument('--hardlink-dupes', action='store_true', dest='hardlink_dupes', help='Hardlink duplicated pyc files') diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 84b25914b0aa93a..5326cf9b19cf883 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -61,8 +61,9 @@ def __setattr__(self, name, _): def get_frame_info(frame_buffer): """Get Zstandard frame information from a frame header. - *frame_buffer* is a bytes-like object. It should start from the beginning - of a frame, and needs to include at least the frame header (6 to 18 bytes). + *frame_buffer* is a bytes-like object. It should start from the + beginning of a frame, and needs to include at least the frame header + (6 to 18 bytes). The returned FrameInfo object has two attributes. 'decompressed_size' is the size in bytes of the data in the frame when @@ -103,16 +104,17 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): finalize *zstd_dict* by adding headers and statistics according to the Zstandard dictionary format. - You may compose an effective dictionary content by hand, which is used as - basis dictionary, and use some samples to finalize a dictionary. The basis - dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. + You may compose an effective dictionary content by hand, which is used + as basis dictionary, and use some samples to finalize a dictionary. The + basis dictionary may be a "raw content" dictionary. See *is_raw* in + ZstdDict. - *samples* is an iterable of samples, where a sample is a bytes-like object - representing a file. + *samples* is an iterable of samples, where a sample is a bytes-like + object representing a file. *dict_size* is the dictionary's maximum size, in bytes. *level* is the expected compression level. The statistics for each - compression level differ, so tuning the dictionary to the compression level - can provide improvements. + compression level differ, so tuning the dictionary to the compression + level can provide improvements. """ if not isinstance(zstd_dict, ZstdDict): @@ -140,8 +142,8 @@ def compress(data, level=None, options=None, zstd_dict=None): COMPRESSION_LEVEL_DEFAULT ('3'). *options* is a dict object that contains advanced compression parameters. See CompressionParameter for more on options. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. For incremental compression, use a ZstdCompressor instead. """ @@ -152,8 +154,8 @@ def compress(data, level=None, options=None, zstd_dict=None): def decompress(data, zstd_dict=None, options=None): """Decompress one or more frames of Zstandard compressed *data*. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. *options* is a dict object that contains advanced compression parameters. See DecompressionParameter for more on options. diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index d709f5efc658fa4..8d3358152e9a889 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -36,9 +36,9 @@ def __init__(self, file, /, mode='r', *, *file* can be either an file-like object, or a file name to open. - *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for - creating exclusively, or 'a' for appending. These can equivalently be - given as 'rb', 'wb', 'xb' and 'ab' respectively. + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' + for creating exclusively, or 'a' for appending. These can + equivalently be given as 'rb', 'wb', 'xb' and 'ab' respectively. *level* is an optional int specifying the compression level to use, or COMPRESSION_LEVEL_DEFAULT if not given. @@ -296,26 +296,27 @@ def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, encoding=None, errors=None, newline=None): """Open a Zstandard compressed file in binary or text mode. - file can be either a file name (given as a str, bytes, or PathLike object), - in which case the named file is opened, or it can be an existing file object - to read from or write to. + file can be either a file name (given as a str, bytes, or PathLike + object), in which case the named file is opened, or it can be + an existing file object to read from or write to. - The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', - 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', + 'a', 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. - The level, options, and zstd_dict parameters specify the settings the same - as ZstdFile. + The level, options, and zstd_dict parameters specify the settings the + same as ZstdFile. When using read mode (decompression), the options parameter is a dict - representing advanced decompression options. The level parameter is not - supported in this case. When using write mode (compression), only one of - level, an int representing the compression level, or options, a dict - representing advanced compression options, may be passed. In both modes, - zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary. - - For binary mode, this function is equivalent to the ZstdFile constructor: - ZstdFile(filename, mode, ...). In this case, the encoding, errors and - newline parameters must not be provided. + representing advanced decompression options. The level parameter is not + supported in this case. When using write mode (compression), only one + of level, an int representing the compression level, or options, a dict + representing advanced compression options, may be passed. In both + modes, zstd_dict is a ZstdDict instance containing a trained Zstandard + dictionary. + + For binary mode, this function is equivalent to the ZstdFile + constructor: ZstdFile(filename, mode, ...). In this case, the encoding, + errors and newline parameters must not be provided. For text mode, an ZstdFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index f506ce68aea5b29..4e71331f9a0a594 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -194,15 +194,15 @@ def as_completed(fs, timeout=None): """An iterator over the given futures that yields each as it completes. Args: - fs: The sequence of Futures (possibly created by different Executors) to - iterate over. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. + fs: The sequence of Futures (possibly created by different + Executors) to iterate over. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. Returns: - An iterator that yields the given Futures as they complete (finished or - cancelled). If any given Futures are duplicated, they will be returned - once. + An iterator that yields the given Futures as they complete + (finished or cancelled). If any given Futures are duplicated, + they will be returned once. Raises: TimeoutError: If the entire result iterator could not be generated @@ -258,19 +258,20 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): """Wait for the futures in the given sequence to complete. Args: - fs: The sequence of Futures (possibly created by different Executors) to - wait upon. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - return_when: Indicates when this function should return. The options - are: + fs: The sequence of Futures (possibly created by different + Executors) to wait upon. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + return_when: Indicates when this function should return. + The options are: FIRST_COMPLETED - Return when any future finishes or is cancelled. FIRST_EXCEPTION - Return when any future finishes by raising an - exception. If no future raises an exception + exception. If no future raises an exception then it is equivalent to ALL_COMPLETED. - ALL_COMPLETED - Return when all futures finish or are cancelled. + ALL_COMPLETED - Return when all futures finish or are + cancelled. Returns: A named 2-tuple of sets. The first set, named 'done', contains the @@ -404,11 +405,12 @@ def add_done_callback(self, fn): Args: fn: A callable that will be called with this future as its only - argument when the future completes or is cancelled. The callable - will always be called by a thread in the same process in which - it was added. If the future has already completed or been - cancelled then the callable will be called immediately. These - callables are called in the order that they were added. + argument when the future completes or is cancelled. The + callable will always be called by a thread in the same + process in which it was added. If the future has already + completed or been cancelled then the callable will be + called immediately. These callables are called in the + order that they were added. """ with self._condition: if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]: @@ -423,17 +425,19 @@ def result(self, timeout=None): """Return the result of the call that the future represents. Args: - timeout: The number of seconds to wait for the result if the future - isn't done. If None, then there is no limit on the wait time. + timeout: The number of seconds to wait for the result if the + future isn't done. If None, then there is no limit on the + wait time. Returns: The result of the call that the future represents. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. - Exception: If the call raised then that exception will be raised. + TimeoutError: If the future didn't finish executing before the + given timeout. + Exception: If the call raised then that exception will be + raised. """ try: with self._condition: @@ -459,17 +463,17 @@ def exception(self, timeout=None): Args: timeout: The number of seconds to wait for the exception if the - future isn't done. If None, then there is no limit on the wait - time. + future isn't done. If None, then there is no limit on the + wait time. Returns: - The exception raised by the call that the future represents or None - if the call completed without raising. + The exception raised by the call that the future represents or + None if the call completed without raising. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. + TimeoutError: If the future didn't finish executing before the + given timeout. """ with self._condition: @@ -494,22 +498,23 @@ def set_running_or_notify_cancel(self): Should only be used by Executor implementations and unit tests. If the future has been cancelled (cancel() was called and returned - True) then any threads waiting on the future completing (though calls - to as_completed() or wait()) are notified and False is returned. + True) then any threads waiting on the future completing (though + calls to as_completed() or wait()) are notified and False is + returned. If the future was not cancelled then it is put in the running state (future calls to running() will return True) and True is returned. This method should be called by Executor implementations before - executing the work associated with this future. If this method returns - False then the work should not be executed. + executing the work associated with this future. If this method + returns False then the work should not be executed. Returns: False if the Future was cancelled, True otherwise. Raises: - RuntimeError: if this method was already called or if set_result() - or set_exception() was called. + RuntimeError: if this method was already called or if + set_result() or set_exception() was called. """ with self._condition: if self._state == CANCELLED: @@ -593,8 +598,9 @@ class Executor(object): def submit(self, fn, /, *args, **kwargs): """Submits a callable to be executed with the given arguments. - Schedules the callable to be executed as fn(*args, **kwargs) and returns - a Future instance representing the execution of the callable. + Schedules the callable to be executed as fn(*args, **kwargs) and + returns a Future instance representing the execution of the + callable. Returns: A Future representing the given call. @@ -607,25 +613,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: The size of the chunks the iterable will be broken into - before being passed to a child process. This argument is only - used by ProcessPoolExecutor; it is ignored by + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: The size of the chunks the iterable will be broken + into before being passed to a child process. This argument + is only used by ProcessPoolExecutor; it is ignored by ThreadPoolExecutor. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if buffersize is not None and not isinstance(buffersize, int): @@ -679,8 +685,8 @@ def shutdown(self, wait=True, *, cancel_futures=False): Args: wait: If True then shutdown will not return until all running - futures have finished executing and the resources used by the - executor have been reclaimed. + futures have finished executing and the resources used by + the executor have been reclaimed. cancel_futures: If True then shutdown will cancel all pending futures. Futures that are completed or running will not be cancelled. diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index 85c1da2c7228943..fd3d45144b49a71 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -110,8 +110,8 @@ def __init__(self, max_workers=None, thread_name_prefix='', """Initializes a new InterpreterPoolExecutor instance. Args: - max_workers: The maximum number of interpreters that can be used to - execute the given calls. + max_workers: The maximum number of interpreters that can be used + to execute the given calls. thread_name_prefix: An optional name prefix to give our threads. initializer: A callable or script used to initialize each worker interpreter. diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a42afa68efcb148..10d4ac89d725713 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -469,11 +469,9 @@ def _terminate_broken(self, cause): executor._shutdown_thread = True executor = None - # All pending tasks are to be marked failed with the following - # BrokenProcessPool error - bpe = BrokenProcessPool("A process in the process pool was " - "terminated abruptly while the future was " - "running or pending.") + # All pending tasks are to be marked failed with a + # BrokenProcessPool error, as separate instances to avoid sharing + # a traceback (gh-101267). cause_str = None if cause is not None: cause_str = ''.join(cause) @@ -489,11 +487,15 @@ def _terminate_broken(self, cause): f"with exit code {p.exitcode}") if errors: cause_str = "\n".join(errors) - if cause_str: - bpe.__cause__ = _RemoteTraceback(f"\n'''\n{cause_str}'''") + cause_tb = f"\n'''\n{cause_str}'''" if cause_str else None # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): + bpe = BrokenProcessPool("A process in the process pool was " + "terminated abruptly while the future was " + "running or pending.") + if cause_tb is not None: + bpe.__cause__ = _RemoteTraceback(cause_tb) try: work_item.future.set_exception(bpe) except _base.InvalidStateError: @@ -656,19 +658,21 @@ def __init__(self, max_workers=None, mp_context=None, Args: max_workers: The maximum number of processes that can be used to - execute the given calls. If None or not given then as many - worker processes will be created as the machine has processors. - mp_context: A multiprocessing context to launch the workers created - using the multiprocessing.get_context('start method') API. This - object should provide SimpleQueue, Queue and Process. + execute the given calls. If None or not given then as many + worker processes will be created as the machine has + processors. + mp_context: A multiprocessing context to launch the workers + created using the multiprocessing.get_context('start method') + API. This object should provide SimpleQueue, Queue and + Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. - max_tasks_per_child: The maximum number of tasks a worker process - can complete before it will exit and be replaced with a fresh - worker process. The default of None means worker process will - live as long as the executor. Requires a non-'fork' mp_context - start method. When given, we default to using 'spawn' if no - mp_context is supplied. + max_tasks_per_child: The maximum number of tasks a worker + process can complete before it will exit and be replaced + with a fresh worker process. The default of None means + worker process will live as long as the executor. Requires + a non-'fork' mp_context start method. When given, we + default to using 'spawn' if no mp_context is supplied. """ _check_system_limits() @@ -838,24 +842,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: If greater than one, the iterables will be chopped into - chunks of size chunksize and submitted to the process pool. - If set to one, the items in the list will be sent one at a time. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: If greater than one, the iterables will be chopped + into chunks of size chunksize and submitted to the process + pool. If set to one, the items in the list will be sent + one at a time. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if chunksize < 1: diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index ee159d7de638272..5f3ee0934de59d6 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -185,7 +185,8 @@ def put(self, obj, block=True, timeout=None, *, underlying data is actually shared. Furthermore, some types can be sent through a queue more efficiently than others. This group includes various immutable types like int, str, bytes, and - tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable(). + tuple (if the items are likewise efficiently shareable). + See interpreters.is_shareable(). "unbounditems" controls the behavior of Queue.get() for the given object if the current interpreter (calling put()) is later diff --git a/Lib/configparser.py b/Lib/configparser.py index a53ac87276445ad..3c452afe8ade485 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -992,7 +992,9 @@ def _write_section(self, fp, section_name, section_items, delimiter, unnamed=Fal value = self._interpolation.before_write(self, section_name, key, value) if value is not None or not self._allow_no_value: - value = delimiter + str(value).replace('\n', '\n\t') + # Convert all possible line-endings into '\n\t' + value = (delimiter + str(value).replace('\r\n', '\n') + .replace('\r', '\n').replace('\n', '\n\t')) else: value = "" fp.write("{}{}\n".format(key, value)) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 0b84a27f8c6d613..890168cc9809fd6 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -549,9 +549,11 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif _sys.platform in ["android", "cygwin"]: +elif _sys.platform == "android": # These are Unix-like platforms which use a dynamically-linked libpython. pythonapi = PyDLL(_sysconfig.get_config_var("LDLIBRARY")) +elif _sys.platform == "cygwin": + pythonapi = PyDLL(_sysconfig.get_config_var("DLLLIBRARY")) else: pythonapi = PyDLL(None) diff --git a/Lib/curses/__init__.py b/Lib/curses/__init__.py index 605d5fcbec5499d..e150c7f932385eb 100644 --- a/Lib/curses/__init__.py +++ b/Lib/curses/__init__.py @@ -34,6 +34,23 @@ def initscr(): setattr(curses, key, value) return stdscr +# newterm() is wrapped for the same reason as initscr(): the ACS_* constants +# and LINES/COLS only become available once a terminal is initialized, and are +# then copied to the curses package's dictionary. + +try: + newterm +except NameError: + pass +else: + def newterm(type=None, fd=None, infd=None, /): + import _curses, curses + screen = _curses.newterm(type, fd, infd) + for key, value in _curses.__dict__.items(): + if key.startswith('ACS_') or key in ('LINES', 'COLS'): + setattr(curses, key, value) + return screen + # This is a similar wrapper for start_color(), which adds the COLORS and # COLOR_PAIRS variables which are only available after start_color() is # called. diff --git a/Lib/curses/textpad.py b/Lib/curses/textpad.py index aa87061b8d749e9..c8dbf9fb614fcbe 100644 --- a/Lib/curses/textpad.py +++ b/Lib/curses/textpad.py @@ -23,22 +23,25 @@ class Textbox: Ctrl-A Go to left edge of window. Ctrl-B Cursor left, wrapping to previous line if appropriate. Ctrl-D Delete character under cursor. - Ctrl-E Go to right edge (stripspaces off) or end of line (stripspaces on). + Ctrl-E Go to right edge (stripspaces off) or end of line + (stripspaces on). Ctrl-F Cursor right, wrapping to next line when appropriate. Ctrl-G Terminate, returning the window contents. Ctrl-H Delete character backward. - Ctrl-J Terminate if the window is 1 line, otherwise insert newline. + Ctrl-J Terminate if the window is 1 line, otherwise move to start + of next line. Ctrl-K If line is blank, delete it, otherwise clear to end of line. Ctrl-L Refresh screen. Ctrl-N Cursor down; move down one line. Ctrl-O Insert a blank line at cursor location. Ctrl-P Cursor up; move up one line. - Move operations do nothing if the cursor is at an edge where the movement - is not possible. The following synonyms are supported where possible: + Move operations do nothing if the cursor is at an edge where the + movement is not possible. The following synonyms are supported where + possible: - KEY_LEFT = Ctrl-B, KEY_RIGHT = Ctrl-F, KEY_UP = Ctrl-P, KEY_DOWN = Ctrl-N - KEY_BACKSPACE = Ctrl-h + KEY_LEFT = Ctrl-B, KEY_RIGHT = Ctrl-F, KEY_UP = Ctrl-P, + KEY_DOWN = Ctrl-N, KEY_BACKSPACE = Ctrl-h """ def __init__(self, win, insert_mode=False): self.win = win diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index d67cc4dd1b19ab5..bd317aa9b0f2f37 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -6,7 +6,6 @@ import abc from reprlib import recursive_repr lazy import copy -lazy import inspect lazy import re @@ -981,6 +980,7 @@ def __get__(self, _obj, cls): try: # In some cases fetching a signature is not possible. # But, we surely should not fail in this case. + import inspect text_sig = str(inspect.signature( cls, annotation_format=annotationlib.Format.FORWARDREF, @@ -1250,6 +1250,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # classes with slots. These could be slightly more performant if we generated # the code instead of iterating over fields. But that can be a project for # another day, if performance becomes an issue. + def _dataclass_getstate(self): return [getattr(self, f.name) for f in fields(self)] @@ -1391,6 +1392,7 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): # If this is a wrapped function, unwrap it. if not isinstance(member, type) and hasattr(member, '__wrapped__'): + import inspect member = inspect.unwrap(member) if isinstance(member, types.FunctionType): @@ -1643,7 +1645,7 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, - weakref_slot=False, module=None, decorator=dataclass): + weakref_slot=False, module=None, qualname=None, decorator=dataclass): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable @@ -1668,6 +1670,9 @@ class C(Base): If module parameter is defined, the '__module__' attribute of the dataclass is set to that value. + + If qualname parameter is defined, the '__qualname__' attribute of the dataclass is set + to that value. """ if namespace is None: @@ -1757,6 +1762,9 @@ def exec_body_callback(ns): if module is not None: cls.__module__ = module + if qualname: + cls.__qualname__ = qualname + # Apply the normal provided decorator. cls = decorator(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, diff --git a/Lib/difflib.py b/Lib/difflib.py index 8f3cdaed9564d87..ae8b284b4d36474 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -30,10 +30,10 @@ 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match'] -from _colorize import can_colorize, get_theme from heapq import nlargest as _nlargest from collections import namedtuple as _namedtuple from types import GenericAlias +lazy from _colorize import can_colorize, get_theme Match = _namedtuple('Match', 'a b size') diff --git a/Lib/dis.py b/Lib/dis.py index 64f3450da3071b8..318729091fa098c 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -84,7 +84,7 @@ def _try_compile(source, name): return compile(source, name, 'exec') def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, - show_offsets=False, show_positions=False): + show_offsets=False, show_positions=False, show_jit=False): """Disassemble classes, methods, functions, and other compiled objects. With no argument, disassemble the last traceback. @@ -95,7 +95,8 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, """ if x is None: distb(file=file, show_caches=show_caches, adaptive=adaptive, - show_offsets=show_offsets, show_positions=show_positions) + show_offsets=show_offsets, show_positions=show_positions, + show_jit=show_jit) return # Extract functions from methods. if hasattr(x, '__func__'): @@ -116,12 +117,14 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, if isinstance(x1, _have_code): print("Disassembly of %s:" % name, file=file) try: - dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, + show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit) except TypeError as msg: print("Sorry:", msg, file=file) print(file=file) elif hasattr(x, 'co_code'): # Code object - _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, + show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit) elif isinstance(x, (bytes, bytearray)): # Raw bytecode labels_map = _make_labels_map(x) label_width = 4 + len(str(len(labels_map))) @@ -132,12 +135,13 @@ def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, arg_resolver = ArgResolver(labels_map=labels_map) _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) elif isinstance(x, str): # Source code - _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, + show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit) else: raise TypeError("don't know how to disassemble %s objects" % type(x).__name__) -def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False, show_jit=False): """Disassemble a traceback (default: last traceback).""" if tb is None: try: @@ -148,7 +152,8 @@ def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets except AttributeError: raise RuntimeError("no last traceback to disassemble") from None while tb.tb_next: tb = tb.tb_next - disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, + show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit) # The inspect module interrogates this dictionary to build its # list of CO_* constants. It is also used by pretty_flags to @@ -216,14 +221,14 @@ def _deoptop(op): name = _all_opname[op] return _all_opmap[deoptmap[name]] if name in deoptmap else op -def _get_code_array(co, adaptive): +def _get_code_array(co, adaptive, show_jit): if adaptive: code = co._co_code_adaptive res = [] found = False for i in range(0, len(code), 2): op, arg = code[i], code[i+1] - if op == ENTER_EXECUTOR: + if op == ENTER_EXECUTOR and not show_jit: try: ex = get_executor(co, i) except (ValueError, RuntimeError): @@ -656,7 +661,7 @@ def get_argval_argrepr(self, op, arg, offset): argrepr = 'not in' if argval else 'in' return argval, argrepr -def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False): +def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False, show_jit=False): """Iterator for the opcodes in methods, functions or code Generates a series of Instruction named tuples giving the details of @@ -679,7 +684,7 @@ def get_instructions(x, *, first_line=None, show_caches=None, adaptive=False): names=co.co_names, varname_from_oparg=co._varname_from_oparg, labels_map=_make_labels_map(original_code)) - return _get_instructions_bytes(_get_code_array(co, adaptive), + return _get_instructions_bytes(_get_code_array(co, adaptive, show_jit), linestarts=linestarts, line_offset=line_offset, co_positions=co.co_positions(), @@ -698,8 +703,8 @@ def _get_const_value(op, arg, co_consts): if op == LOAD_SMALL_INT: return arg if op == LOAD_COMMON_CONSTANT: - # Opargs 0-6 are callables; 7-11 are literal values. - if 7 <= arg <= 11: + # Opargs 0-6 and 12 are callables; 7-11 and 13 are literal values. + if 7 <= arg <= 11 or arg == 13: return _common_constants[arg] return UNKNOWN argval = UNKNOWN @@ -792,6 +797,8 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N positions = Positions(*next(co_positions, ())) deop = _deoptop(op) op = code[offset] + if op == ENTER_EXECUTOR: + arg = code[offset+1] if arg_resolver: argval, argrepr = arg_resolver.get_argval_argrepr(op, arg, offset) @@ -820,7 +827,7 @@ def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=N def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, - show_offsets=False, show_positions=False): + show_offsets=False, show_positions=False, show_jit=False): """Disassemble a code object.""" linestarts = dict(findlinestarts(co)) exception_entries = _parse_exception_table(co) @@ -840,12 +847,12 @@ def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, names=co.co_names, varname_from_oparg=co._varname_from_oparg, labels_map=labels_map) - _disassemble_bytes(_get_code_array(co, adaptive), lasti, linestarts, + _disassemble_bytes(_get_code_array(co, adaptive, show_jit), lasti, linestarts, exception_entries=exception_entries, co_positions=co.co_positions(), original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) -def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): - disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions) +def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False, show_jit=False): + disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions, show_jit=show_jit) if depth is None or depth > 0: if depth is not None: depth = depth - 1 @@ -855,7 +862,8 @@ def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adap print("Disassembly of %r:" % (x,), file=file) _disassemble_recursive( x, file=file, depth=depth, show_caches=show_caches, - adaptive=adaptive, show_offsets=show_offsets, show_positions=show_positions + adaptive=adaptive, show_offsets=show_offsets, + show_positions=show_positions, show_jit=show_jit ) @@ -1054,7 +1062,7 @@ class Bytecode: Iterating over this yields the bytecode operations as Instruction instances. """ - def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False): + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False, show_positions=False, show_jit=False): self.codeobj = co = _get_code_object(x) if first_line is None: self.first_line = co.co_firstlineno @@ -1070,6 +1078,7 @@ def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False self.adaptive = adaptive self.show_offsets = show_offsets self.show_positions = show_positions + self.show_jit = show_jit def __iter__(self): co = self.codeobj @@ -1079,7 +1088,7 @@ def __iter__(self): names=co.co_names, varname_from_oparg=co._varname_from_oparg, labels_map=labels_map) - return _get_instructions_bytes(_get_code_array(co, self.adaptive), + return _get_instructions_bytes(_get_code_array(co, self.adaptive, self.show_jit), linestarts=self._linestarts, line_offset=self._line_offset, co_positions=co.co_positions(), @@ -1111,7 +1120,7 @@ def dis(self): else: offset = -1 with io.StringIO() as output: - code = _get_code_array(co, self.adaptive) + code = _get_code_array(co, self.adaptive, self.show_jit) offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 if self.show_positions: lineno_width = _get_positions_width(co) @@ -1146,7 +1155,7 @@ def dis(self): def main(args=None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-C', '--show-caches', action='store_true', help='show inline caches') parser.add_argument('-O', '--show-offsets', action='store_true', diff --git a/Lib/doctest.py b/Lib/doctest.py index 0fcfa1e3e971446..8a55fe3ddd26154 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -106,8 +106,8 @@ def _test(): import unittest from io import StringIO, TextIOWrapper, BytesIO from collections import namedtuple -import _colorize # Used in doctests -from _colorize import ANSIColors, can_colorize +lazy import _colorize # Used in doctests +lazy from _colorize import ANSIColors, can_colorize class TestResults(namedtuple('TestResults', 'failed attempted')): @@ -1768,7 +1768,7 @@ def check_output(self, want, got, optionflags): '', want) # If a line in got contains only spaces, then remove the # spaces. - got = re.sub(r'(?m)^[^\S\n]+$', '', got) + got = re.sub(r'(?m)^[\s--\n]+$', '', got) if got == want: return True @@ -2951,7 +2951,7 @@ def get(self): def _test(): import argparse - parser = argparse.ArgumentParser(description="doctest runner", color=True) + parser = argparse.ArgumentParser(description="doctest runner") parser.add_argument('-v', '--verbose', action='store_true', default=False, help='print very verbose output for all tests') parser.add_argument('-o', '--option', action='append', @@ -2961,8 +2961,8 @@ def _test(): ' than once to apply multiple options')) parser.add_argument('-f', '--fail-fast', action='store_true', help=('stop running tests after first failure (this' - ' is a shorthand for -o FAIL_FAST, and is' - ' in addition to any other -o options)')) + ' is a shorthand for `-o FAIL_FAST`, and is' + ' in addition to any other `-o` options)')) parser.add_argument('file', nargs='+', help='file containing the tests to run') args = parser.parse_args() diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9873958f5c2790c..792072ab9f6128a 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1461,6 +1461,16 @@ def get_phrase(value): else: try: token, value = get_word(value) + if (token[0].token_type == 'encoded-word' + and phrase + and phrase[-1].token_type == 'atom' + and len(phrase[-1]) > 1 + and phrase[-1][-2].token_type == 'encoded-word' + and phrase[-1][-1].token_type == 'cfws' + and not phrase[-1][-1].comments + ): + # linear ws between ews needs special handing... + phrase[-1][-1] = EWWhiteSpaceTerminal(phrase[-1], 'fws') except errors.HeaderParseError: if value[0] in CFWS_LEADER: token, value = get_cfws(value) diff --git a/Lib/email/_parseaddr.py b/Lib/email/_parseaddr.py index 6a7c5fa06d20b6e..e311948cb09a7d6 100644 --- a/Lib/email/_parseaddr.py +++ b/Lib/email/_parseaddr.py @@ -59,7 +59,7 @@ def _parsedate_tz(data): The last (additional) element is the time zone offset in seconds, except if the timezone was specified as -0000. In that case the last element is - None. This indicates a UTC timestamp that explicitly declaims knowledge of + None. This indicates a UTC timestamp that explicitly disclaims knowledge of the source timezone, as opposed to a +0000 timestamp that indicates the source timezone really was UTC. diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 5036c3f58a5633c..5981791820e740c 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -9,6 +9,7 @@ 'add_codec', ] +import codecs from functools import partial import email.base64mime @@ -58,43 +59,75 @@ 'shift_jis': (BASE64, None, 'iso-2022-jp'), 'iso-2022-jp': (BASE64, None, None), 'koi8-r': (BASE64, BASE64, None), - 'utf-8': (SHORTEST, BASE64, 'utf-8'), } -# Aliases for other commonly-used names for character sets. Map -# them to the real ones used in email. +# Map Python codec names to their corresponding MIME/IANA names. ALIASES = { - 'latin_1': 'iso-8859-1', - 'latin-1': 'iso-8859-1', - 'latin_2': 'iso-8859-2', - 'latin-2': 'iso-8859-2', - 'latin_3': 'iso-8859-3', - 'latin-3': 'iso-8859-3', - 'latin_4': 'iso-8859-4', - 'latin-4': 'iso-8859-4', - 'latin_5': 'iso-8859-9', - 'latin-5': 'iso-8859-9', - 'latin_6': 'iso-8859-10', - 'latin-6': 'iso-8859-10', - 'latin_7': 'iso-8859-13', - 'latin-7': 'iso-8859-13', - 'latin_8': 'iso-8859-14', - 'latin-8': 'iso-8859-14', - 'latin_9': 'iso-8859-15', - 'latin-9': 'iso-8859-15', - 'latin_10':'iso-8859-16', - 'latin-10':'iso-8859-16', - 'cp949': 'ks_c_5601-1987', - 'euc_jp': 'euc-jp', - 'euc_kr': 'euc-kr', - 'ascii': 'us-ascii', - } + 'ascii': 'us-ascii', + 'big5hkscs': 'big5-hkscs', + 'cp037': 'ibm037', + 'cp1026': 'ibm1026', + 'cp1140': 'ibm01140', + 'cp1250': 'windows-1250', + 'cp1251': 'windows-1251', + 'cp1252': 'windows-1252', + 'cp1253': 'windows-1253', + 'cp1254': 'windows-1254', + 'cp1255': 'windows-1255', + 'cp1256': 'windows-1256', + 'cp1257': 'windows-1257', + 'cp1258': 'windows-1258', + 'cp273': 'ibm273', + 'cp424': 'ibm424', + 'cp437': 'ibm437', + 'cp500': 'ibm500', + 'cp775': 'ibm775', + 'cp850': 'ibm850', + 'cp852': 'ibm852', + 'cp855': 'ibm855', + 'cp857': 'ibm857', + 'cp858': 'ibm00858', + 'cp860': 'ibm860', + 'cp861': 'ibm861', + 'cp862': 'ibm862', + 'cp863': 'ibm863', + 'cp864': 'ibm864', + 'cp865': 'ibm865', + 'cp866': 'ibm866', + 'cp869': 'ibm869', + 'cp874': 'windows-874', + 'euc_jp': 'euc-jp', + 'euc_kr': 'euc-kr', + 'hz': 'hz-gb-2312', + 'iso2022_jp': 'iso-2022-jp', + 'iso2022_jp_2': 'iso-2022-jp-2', + 'iso2022_kr': 'iso-2022-kr', + 'iso8859-1': 'iso-8859-1', + 'iso8859-10': 'iso-8859-10', + 'iso8859-11': 'iso-8859-11', + 'iso8859-13': 'iso-8859-13', + 'iso8859-14': 'iso-8859-14', + 'iso8859-15': 'iso-8859-15', + 'iso8859-16': 'iso-8859-16', + 'iso8859-2': 'iso-8859-2', + 'iso8859-3': 'iso-8859-3', + 'iso8859-4': 'iso-8859-4', + 'iso8859-5': 'iso-8859-5', + 'iso8859-6': 'iso-8859-6', + 'iso8859-7': 'iso-8859-7', + 'iso8859-8': 'iso-8859-8-i', + 'iso8859-9': 'iso-8859-9', + 'kz1048': 'kz-1048', + 'mac-roman': 'macintosh', + + # CP949 is not registered in IANA. KS_C_5601-1987 is not the same, + # but the closest registered option. + 'cp949': 'ks_c_5601-1987', +} # Map charsets to their Unicode codec strings. CODEC_MAP = { - 'gb2312': 'eucgb2312_cn', - 'big5': 'big5_tw', # Hack: We don't want *any* conversion for stuff marked us-ascii, as all # sorts of garbage might be sent to us in the guise of 7-bit us-ascii. # Let that stuff pass through without conversion to/from Unicode. @@ -217,7 +250,18 @@ def __init__(self, input_charset=DEFAULT_CHARSET): raise errors.CharsetError(input_charset) input_charset = input_charset.lower() # Set the input charset after filtering through the aliases - self.input_charset = ALIASES.get(input_charset, input_charset) + # For backward compatibility, try ALIASES first to let the user + # override it. + if input_charset in ALIASES: + input_charset = ALIASES[input_charset] + else: + try: + input_codec = codecs.lookup(input_charset).name + except LookupError: + pass + else: + input_charset = ALIASES.get(input_codec, input_codec) + self.input_charset = input_charset # We can try to guess which encoding and conversion to use by the # charset_map dictionary. Try that first, but let the user override # it. diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index 13fcb9787f1f320..c0090af716575d7 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -173,11 +173,12 @@ def set_text_content(msg, string, subtype="plain", charset='utf-8', cte=None, disposition=None, filename=None, cid=None, params=None, headers=None): _prepare_set(msg, 'text', subtype, headers) + + cs = email.charset.Charset(charset) + charset = cs.output_charset cte, payload = _encode_text(string, charset, cte, msg.policy) msg.set_payload(payload) - msg.set_param('charset', - email.charset.ALIASES.get(charset, charset), - replace=True) + msg.set_param('charset', charset, replace=True) msg['Content-Transfer-Encoding'] = cte _finalize_set(msg, disposition, filename, cid, params) raw_data_manager.add_set_handler(str, set_text_content) diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index f4b1b8dd43f9205..ef51168d755ba98 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -43,6 +43,7 @@ # big5hkscs codec 'big5_hkscs' : 'big5hkscs', + 'csbig5hkscs' : 'big5hkscs', 'hkscs' : 'big5hkscs', # bz2_codec codec @@ -71,42 +72,56 @@ # cp1140 codec '1140' : 'cp1140', + 'ccsid01140' : 'cp1140', + 'cp01140' : 'cp1140', + 'csibm01140' : 'cp1140', + 'ebcdic_us_37_euro' : 'cp1140', + 'ibm01140' : 'cp1140', 'ibm1140' : 'cp1140', # cp1250 codec '1250' : 'cp1250', + 'cswindows1250' : 'cp1250', 'windows_1250' : 'cp1250', # cp1251 codec '1251' : 'cp1251', + 'cswindows1251' : 'cp1251', 'windows_1251' : 'cp1251', # cp1252 codec '1252' : 'cp1252', + 'cswindows1252' : 'cp1252', 'windows_1252' : 'cp1252', # cp1253 codec '1253' : 'cp1253', + 'cswindows1253' : 'cp1253', 'windows_1253' : 'cp1253', # cp1254 codec '1254' : 'cp1254', + 'cswindows1254' : 'cp1254', 'windows_1254' : 'cp1254', # cp1255 codec '1255' : 'cp1255', + 'cswindows1255' : 'cp1255', 'windows_1255' : 'cp1255', # cp1256 codec '1256' : 'cp1256', + 'cswindows1256' : 'cp1256', 'windows_1256' : 'cp1256', # cp1257 codec '1257' : 'cp1257', + 'cswindows1257' : 'cp1257', 'windows_1257' : 'cp1257', # cp1258 codec '1258' : 'cp1258', + 'cswindows1258' : 'cp1258', 'windows_1258' : 'cp1258', # cp273 codec @@ -159,8 +174,13 @@ # cp858 codec '858' : 'cp858', + 'ccsid00858' : 'cp858', + 'cp00858' : 'cp858', + 'csibm00858' : 'cp858', 'csibm858' : 'cp858', + 'ibm00858' : 'cp858', 'ibm858' : 'cp858', + 'pc_multilingual_850_euro' : 'cp858', # cp860 codec '860' : 'cp860', @@ -206,11 +226,13 @@ # cp874 codec '874' : 'cp874', + 'cswindows874' : 'cp874', 'ms874' : 'cp874', 'windows_874' : 'cp874', # cp932 codec '932' : 'cp932', + 'cswindows31j' : 'cp932', 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', @@ -218,6 +240,16 @@ # cp949 codec '949' : 'cp949', + 'csksc56011987' : 'cp949', + 'iso_ir_149' : 'cp949', + 'korean' : 'cp949', + 'ks_c_5601_1987' : 'cp949', + 'ks_c_5601_1989' : 'cp949', + 'ksc5601' : 'cp949', + 'ks_c_5601' : 'cp949', + 'ksc_5601' : 'cp949', + 'ksx1001' : 'cp949', + 'ks_x_1001' : 'cp949', 'ms949' : 'cp949', 'uhc' : 'cp949', @@ -234,47 +266,47 @@ 'eucjisx0213' : 'euc_jisx0213', # euc_jp codec + 'cseucpkdfmtjapanese' : 'euc_jp', 'eucjp' : 'euc_jp', 'ujis' : 'euc_jp', 'u_jis' : 'euc_jp', # euc_kr codec - 'euckr' : 'euc_kr', - 'korean' : 'euc_kr', - 'ksc5601' : 'euc_kr', - 'ks_c_5601' : 'euc_kr', - 'ks_c_5601_1987' : 'euc_kr', - 'ksx1001' : 'euc_kr', - 'ks_x_1001' : 'euc_kr', 'cseuckr' : 'euc_kr', + 'euckr' : 'euc_kr', # gb18030 codec + 'csgb18030' : 'gb18030', 'gb18030_2000' : 'gb18030', # gb2312 codec 'chinese' : 'gb2312', + 'csgb2312' : 'gb2312', 'csiso58gb231280' : 'gb2312', 'euc_cn' : 'gb2312', 'euccn' : 'gb2312', 'eucgb2312_cn' : 'gb2312', 'gb2312_1980' : 'gb2312', 'gb2312_80' : 'gb2312', + 'gb_2312_80' : 'gb2312', 'iso_ir_58' : 'gb2312', # gbk codec '936' : 'gbk', 'cp936' : 'gbk', + 'csgbk' : 'gbk', 'ms936' : 'gbk', + 'windows_936' : 'gbk', # hex_codec codec 'hex' : 'hex_codec', # hp_roman8 codec - 'roman8' : 'hp_roman8', - 'r8' : 'hp_roman8', - 'csHPRoman8' : 'hp_roman8', 'cp1051' : 'hp_roman8', + 'cshproman8' : 'hp_roman8', 'ibm1051' : 'hp_roman8', + 'r8' : 'hp_roman8', + 'roman8' : 'hp_roman8', # hz codec 'hzgb' : 'hz', @@ -291,6 +323,7 @@ 'iso_2022_jp_1' : 'iso2022_jp_1', # iso2022_jp_2 codec + 'csiso2022jp2' : 'iso2022_jp_2', 'iso2022jp_2' : 'iso2022_jp_2', 'iso_2022_jp_2' : 'iso2022_jp_2', @@ -326,12 +359,14 @@ 'iso_8859_11_2001' : 'iso8859_11', # iso8859_13 codec + 'csiso885913' : 'iso8859_13', 'iso_8859_13' : 'iso8859_13', 'l7' : 'iso8859_13', 'latin7' : 'iso8859_13', 'latin_7' : 'iso8859_13', # iso8859_14 codec + 'csiso885914' : 'iso8859_14', 'iso_8859_14' : 'iso8859_14', 'iso_8859_14_1998' : 'iso8859_14', 'iso_celtic' : 'iso8859_14', @@ -341,12 +376,14 @@ 'latin_8' : 'iso8859_14', # iso8859_15 codec + 'csiso885915' : 'iso8859_15', 'iso_8859_15' : 'iso8859_15', 'l9' : 'iso8859_15', 'latin9' : 'iso8859_15', 'latin_9' : 'iso8859_15', # iso8859_16 codec + 'csiso885916' : 'iso8859_16', 'iso_8859_16' : 'iso8859_16', 'iso_8859_16_2001' : 'iso8859_16', 'iso_ir_226' : 'iso8859_16', @@ -408,6 +445,8 @@ 'iso_ir_126' : 'iso8859_7', # iso8859_8 codec + 'csiso88598e' : 'iso8859_8', + 'csiso88598i' : 'iso8859_8', 'csisolatinhebrew' : 'iso8859_8', 'hebrew' : 'iso8859_8', 'iso_8859_8' : 'iso8859_8', @@ -432,7 +471,11 @@ # koi8_r codec 'cskoi8r' : 'koi8_r', + # koi8_u codec + 'cskoi8u' : 'koi8_u', + # kz1048 codec + 'cskz1048' : 'kz1048', 'kz_1048' : 'kz1048', 'rk1048' : 'kz1048', 'strk1048_2002' : 'kz1048', @@ -472,7 +515,9 @@ 'maclatin2' : 'mac_latin2', # mac_roman codec + 'csmacintosh' : 'mac_roman', 'macintosh' : 'mac_roman', + 'mac' : 'mac_roman', 'macroman' : 'mac_roman', # mac_turkish codec @@ -513,6 +558,7 @@ 's_jisx0213' : 'shift_jisx0213', # tis_620 codec + 'cstis620' : 'tis_620', 'tis620' : 'tis_620', 'tis_620_0' : 'tis_620', 'tis_620_2529_0' : 'tis_620', @@ -520,33 +566,42 @@ 'iso_ir_166' : 'tis_620', # utf_16 codec + 'csutf16' : 'utf_16', 'u16' : 'utf_16', 'utf16' : 'utf_16', # utf_16_be codec + 'csutf16be' : 'utf_16_be', 'unicodebigunmarked' : 'utf_16_be', 'utf_16be' : 'utf_16_be', # utf_16_le codec + 'csutf16le' : 'utf_16_le', 'unicodelittleunmarked' : 'utf_16_le', 'utf_16le' : 'utf_16_le', # utf_32 codec + 'csutf32' : 'utf_32', 'u32' : 'utf_32', 'utf32' : 'utf_32', # utf_32_be codec + 'csutf32be' : 'utf_32_be', 'utf_32be' : 'utf_32_be', # utf_32_le codec + 'csutf32le' : 'utf_32_le', 'utf_32le' : 'utf_32_le', # utf_7 codec + 'csunicode11utf7' : 'utf_7', + 'csutf7' : 'utf_7', 'u7' : 'utf_7', 'utf7' : 'utf_7', 'unicode_1_1_utf_7' : 'utf_7', # utf_8 codec + 'csutf8' : 'utf_8', 'u8' : 'utf_8', 'utf' : 'utf_8', 'utf8' : 'utf_8', diff --git a/Lib/encodings/big5.py b/Lib/encodings/big5.py index 7adeb0e16052744..4f749507d78b8b6 100644 --- a/Lib/encodings/big5.py +++ b/Lib/encodings/big5.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -1, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, -1), ) diff --git a/Lib/encodings/big5hkscs.py b/Lib/encodings/big5hkscs.py index 350df37baaedafc..a88caa5e1404c0e 100644 --- a/Lib/encodings/big5hkscs.py +++ b/Lib/encodings/big5hkscs.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1), ) diff --git a/Lib/encodings/cp932.py b/Lib/encodings/cp932.py index e01f59b71905765..86e6ffe3b16c4f8 100644 --- a/Lib/encodings/cp932.py +++ b/Lib/encodings/cp932.py @@ -36,4 +36,18 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + 0x80, -2, -2, -2, -2, -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + 0xf8f0, 0xff61, 0xff62, 0xff63, 0xff64, 0xff65, 0xff66, 0xff67, + 0xff68, 0xff69, 0xff6a, 0xff6b, 0xff6c, 0xff6d, 0xff6e, 0xff6f, + 0xff70, 0xff71, 0xff72, 0xff73, 0xff74, 0xff75, 0xff76, 0xff77, + 0xff78, 0xff79, 0xff7a, 0xff7b, 0xff7c, 0xff7d, 0xff7e, 0xff7f, + 0xff80, 0xff81, 0xff82, 0xff83, 0xff84, 0xff85, 0xff86, 0xff87, + 0xff88, 0xff89, 0xff8a, 0xff8b, 0xff8c, 0xff8d, 0xff8e, 0xff8f, + 0xff90, 0xff91, 0xff92, 0xff93, 0xff94, 0xff95, 0xff96, 0xff97, + 0xff98, 0xff99, 0xff9a, 0xff9b, 0xff9c, 0xff9d, 0xff9e, 0xff9f, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -2, -2, -1, + -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -1, -1, -1, 0xf8f1, 0xf8f2, 0xf8f3), ) diff --git a/Lib/encodings/cp949.py b/Lib/encodings/cp949.py index 627c87125e2affe..7283dba8dbb8d32 100644 --- a/Lib/encodings/cp949.py +++ b/Lib/encodings/cp949.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1), ) diff --git a/Lib/encodings/cp950.py b/Lib/encodings/cp950.py index 39eec5ed0ddef95..d530f914880a32a 100644 --- a/Lib/encodings/cp950.py +++ b/Lib/encodings/cp950.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -1, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, -1), ) diff --git a/Lib/encodings/euc_jis_2004.py b/Lib/encodings/euc_jis_2004.py index 72b87aea68862f2..557f926b8cdb97c 100644 --- a/Lib/encodings/euc_jis_2004.py +++ b/Lib/encodings/euc_jis_2004.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1), ) diff --git a/Lib/encodings/euc_jisx0213.py b/Lib/encodings/euc_jisx0213.py index cc47d04112a187d..bace554431ba3a6 100644 --- a/Lib/encodings/euc_jisx0213.py +++ b/Lib/encodings/euc_jisx0213.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1), ) diff --git a/Lib/encodings/euc_jp.py b/Lib/encodings/euc_jp.py index 7bcbe4147f2ad49..b8df1bc0e2d5fb8 100644 --- a/Lib/encodings/euc_jp.py +++ b/Lib/encodings/euc_jp.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -3, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, -1, -1, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1), ) diff --git a/Lib/encodings/euc_kr.py b/Lib/encodings/euc_kr.py index c1fb1260e879f0c..ee54e17180b5e15 100644 --- a/Lib/encodings/euc_kr.py +++ b/Lib/encodings/euc_kr.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/gb18030.py b/Lib/encodings/gb18030.py index 34fb6c366a76147..c2269a7a98105c4 100644 --- a/Lib/encodings/gb18030.py +++ b/Lib/encodings/gb18030.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/gb2312.py b/Lib/encodings/gb2312.py index 3c3b837d618ecd2..0a9313b05bd75ff 100644 --- a/Lib/encodings/gb2312.py +++ b/Lib/encodings/gb2312.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, -1, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, -1, -1, -1), ) diff --git a/Lib/encodings/gbk.py b/Lib/encodings/gbk.py index 1b45db89859cdf5..45e38bba391533a 100644 --- a/Lib/encodings/gbk.py +++ b/Lib/encodings/gbk.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1), ) diff --git a/Lib/encodings/hz.py b/Lib/encodings/hz.py index 383442a3c9ac9ae..f17f32e0f6f64c4 100644 --- a/Lib/encodings/hz.py +++ b/Lib/encodings/hz.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index d31ee07ab45b76d..c896ffdeadfef7e 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -385,4 +385,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/iso2022_jp.py b/Lib/encodings/iso2022_jp.py index ab0406069356e4d..40892f5c151d4d6 100644 --- a/Lib/encodings/iso2022_jp.py +++ b/Lib/encodings/iso2022_jp.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/iso2022_jp_1.py b/Lib/encodings/iso2022_jp_1.py index 997044dc3787496..98210d617879e2f 100644 --- a/Lib/encodings/iso2022_jp_1.py +++ b/Lib/encodings/iso2022_jp_1.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/iso2022_jp_2.py b/Lib/encodings/iso2022_jp_2.py index 9106bf762512fdd..047cd7c9677c54c 100644 --- a/Lib/encodings/iso2022_jp_2.py +++ b/Lib/encodings/iso2022_jp_2.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/iso2022_jp_2004.py b/Lib/encodings/iso2022_jp_2004.py index 40198bf098570be..9b29edacce3fea4 100644 --- a/Lib/encodings/iso2022_jp_2004.py +++ b/Lib/encodings/iso2022_jp_2004.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/iso2022_jp_3.py b/Lib/encodings/iso2022_jp_3.py index 346e08beccbbafd..a39de6301ccdece 100644 --- a/Lib/encodings/iso2022_jp_3.py +++ b/Lib/encodings/iso2022_jp_3.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/iso2022_jp_ext.py b/Lib/encodings/iso2022_jp_ext.py index 752bab9813a094e..b7470ec98936559 100644 --- a/Lib/encodings/iso2022_jp_ext.py +++ b/Lib/encodings/iso2022_jp_ext.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/iso2022_kr.py b/Lib/encodings/iso2022_kr.py index bf7018763eae387..48dff8dc68e85c8 100644 --- a/Lib/encodings/iso2022_kr.py +++ b/Lib/encodings/iso2022_kr.py @@ -36,4 +36,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/johab.py b/Lib/encodings/johab.py index 512aeeb732b5221..99c9cf6335aaf0f 100644 --- a/Lib/encodings/johab.py +++ b/Lib/encodings/johab.py @@ -36,4 +36,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -1, -1, -1, -1, -1, -2, -2, -2, -2, -2, -2, -1, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, -1), ) diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 268fccbd53974e5..279245f435e1791 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -250,4 +250,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/raw_unicode_escape.py b/Lib/encodings/raw_unicode_escape.py index 46c8e070dd192ed..911f59ccf9a1dd5 100644 --- a/Lib/encodings/raw_unicode_escape.py +++ b/Lib/encodings/raw_unicode_escape.py @@ -43,4 +43,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/shift_jis.py b/Lib/encodings/shift_jis.py index 83381172764dea0..4b33b6fd93cbbe2 100644 --- a/Lib/encodings/shift_jis.py +++ b/Lib/encodings/shift_jis.py @@ -36,4 +36,17 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -2, -2, -2, -2, -1, -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -1, 0xff61, 0xff62, 0xff63, 0xff64, 0xff65, 0xff66, 0xff67, + 0xff68, 0xff69, 0xff6a, 0xff6b, 0xff6c, 0xff6d, 0xff6e, 0xff6f, + 0xff70, 0xff71, 0xff72, 0xff73, 0xff74, 0xff75, 0xff76, 0xff77, + 0xff78, 0xff79, 0xff7a, 0xff7b, 0xff7c, 0xff7d, 0xff7e, 0xff7f, + 0xff80, 0xff81, 0xff82, 0xff83, 0xff84, 0xff85, 0xff86, 0xff87, + 0xff88, 0xff89, 0xff8a, 0xff8b, 0xff8c, 0xff8d, 0xff8e, 0xff8f, + 0xff90, 0xff91, 0xff92, 0xff93, 0xff94, 0xff95, 0xff96, 0xff97, + 0xff98, 0xff99, 0xff9a, 0xff9b, 0xff9c, 0xff9d, 0xff9e, 0xff9f, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1), ) diff --git a/Lib/encodings/shift_jis_2004.py b/Lib/encodings/shift_jis_2004.py index 161b1e86f9918a4..195519eddf50f39 100644 --- a/Lib/encodings/shift_jis_2004.py +++ b/Lib/encodings/shift_jis_2004.py @@ -36,4 +36,18 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=( + *range(0x5c), 0xa5, *range(0x5d, 0x7e), 0x203e, 0x7f, + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -1, 0xff61, 0xff62, 0xff63, 0xff64, 0xff65, 0xff66, 0xff67, + 0xff68, 0xff69, 0xff6a, 0xff6b, 0xff6c, 0xff6d, 0xff6e, 0xff6f, + 0xff70, 0xff71, 0xff72, 0xff73, 0xff74, 0xff75, 0xff76, 0xff77, + 0xff78, 0xff79, 0xff7a, 0xff7b, 0xff7c, 0xff7d, 0xff7e, 0xff7f, + 0xff80, 0xff81, 0xff82, 0xff83, 0xff84, 0xff85, 0xff86, 0xff87, + 0xff88, 0xff89, 0xff8a, 0xff8b, 0xff8c, 0xff8d, 0xff8e, 0xff8f, + 0xff90, 0xff91, 0xff92, 0xff93, 0xff94, 0xff95, 0xff96, 0xff97, + 0xff98, 0xff99, 0xff9a, 0xff9b, 0xff9c, 0xff9d, 0xff9e, 0xff9f, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1), ) diff --git a/Lib/encodings/shift_jisx0213.py b/Lib/encodings/shift_jisx0213.py index cb653f53055e679..b533eed6c18cbfb 100644 --- a/Lib/encodings/shift_jisx0213.py +++ b/Lib/encodings/shift_jisx0213.py @@ -36,4 +36,18 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=( + *range(0x5c), 0xa5, *range(0x5d, 0x7e), 0x203e, 0x7f, + -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -1, 0xff61, 0xff62, 0xff63, 0xff64, 0xff65, 0xff66, 0xff67, + 0xff68, 0xff69, 0xff6a, 0xff6b, 0xff6c, 0xff6d, 0xff6e, 0xff6f, + 0xff70, 0xff71, 0xff72, 0xff73, 0xff74, 0xff75, 0xff76, 0xff77, + 0xff78, 0xff79, 0xff7a, 0xff7b, 0xff7c, 0xff7d, 0xff7e, 0xff7f, + 0xff80, 0xff81, 0xff82, 0xff83, 0xff84, 0xff85, 0xff86, 0xff87, + 0xff88, 0xff89, 0xff8a, 0xff8b, 0xff8c, 0xff8d, 0xff8e, 0xff8f, + 0xff90, 0xff91, 0xff92, 0xff93, 0xff94, 0xff95, 0xff96, 0xff97, + 0xff98, 0xff99, 0xff9a, 0xff9b, 0xff9c, 0xff9d, 0xff9e, 0xff9f, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -1, -1, -1), ) diff --git a/Lib/encodings/unicode_escape.py b/Lib/encodings/unicode_escape.py index 9b1ce99b339ae08..52e4dc256ce7ff7 100644 --- a/Lib/encodings/unicode_escape.py +++ b/Lib/encodings/unicode_escape.py @@ -43,4 +43,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamwriter=StreamWriter, streamreader=StreamReader, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py index d3b9980026666f1..01853d46c89bf0f 100644 --- a/Lib/encodings/utf_16.py +++ b/Lib/encodings/utf_16.py @@ -152,4 +152,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/utf_16_be.py b/Lib/encodings/utf_16_be.py index 86b458eb9bcd96b..c4f8753e999b903 100644 --- a/Lib/encodings/utf_16_be.py +++ b/Lib/encodings/utf_16_be.py @@ -39,4 +39,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/utf_16_le.py b/Lib/encodings/utf_16_le.py index ec454142eedf251..aa68f019f9ea2cf 100644 --- a/Lib/encodings/utf_16_le.py +++ b/Lib/encodings/utf_16_le.py @@ -39,4 +39,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py index 1924bedbb74c686..446503ccb32ee0f 100644 --- a/Lib/encodings/utf_32.py +++ b/Lib/encodings/utf_32.py @@ -147,4 +147,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/utf_32_be.py b/Lib/encodings/utf_32_be.py index fe272b5fafec69e..c430c7ee0ac8977 100644 --- a/Lib/encodings/utf_32_be.py +++ b/Lib/encodings/utf_32_be.py @@ -34,4 +34,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/utf_32_le.py b/Lib/encodings/utf_32_le.py index 9e48210928ee652..7fb33289054770a 100644 --- a/Lib/encodings/utf_32_le.py +++ b/Lib/encodings/utf_32_le.py @@ -34,4 +34,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/utf_7.py b/Lib/encodings/utf_7.py index 8e0567f2087d65e..9f70aaff4f7a7f2 100644 --- a/Lib/encodings/utf_7.py +++ b/Lib/encodings/utf_7.py @@ -35,4 +35,5 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=False, ) diff --git a/Lib/encodings/utf_8.py b/Lib/encodings/utf_8.py index 1bf6336571547bc..854cb88375c37f3 100644 --- a/Lib/encodings/utf_8.py +++ b/Lib/encodings/utf_8.py @@ -39,4 +39,13 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -4, -4, -4, -4, -4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1), ) diff --git a/Lib/encodings/utf_8_sig.py b/Lib/encodings/utf_8_sig.py index 1bb479203f365da..cc895cddcc561bc 100644 --- a/Lib/encodings/utf_8_sig.py +++ b/Lib/encodings/utf_8_sig.py @@ -127,4 +127,14 @@ def getregentry(): incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, + # The same as for UTF-8. + _expat_decoding_table=(*range(128), + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, + -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, -3, + -4, -4, -4, -4, -4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1), ) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 5a55525d6bd235d..e14d23b35cf7a95 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "26.1" +_PIP_VERSION = "26.1.2" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora @@ -217,7 +217,7 @@ def _uninstall_helper(*, verbosity=0): def _main(argv=None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument( "--version", action="version", diff --git a/Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-26.1.2-py3-none-any.whl similarity index 92% rename from Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-26.1.2-py3-none-any.whl index b51afa14f7c0ad8..24b5cc90ace6439 100644 Binary files a/Lib/ensurepip/_bundled/pip-26.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-26.1.2-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 025e973446d88d0..7aff36c94ce1dcf 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -702,9 +702,9 @@ def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, """ Either returns an existing member, or creates a new enum class. - This method is used both when an enum class is given a value to match - to an enumeration member (i.e. Color(3)) and for the functional API - (i.e. Color = Enum('Color', names='RED GREEN BLUE')). + This method is used both when an enum class is given a value to + match to an enumeration member (i.e. Color(3)) and for the + functional API (i.e. Color = Enum('Color', names='RED GREEN BLUE')). The value lookup branch is chosen if the enum is final. @@ -712,16 +712,17 @@ def __call__(cls, value, names=_not_given, *values, module=None, qualname=None, `value` will be the name of the new class. - `names` should be either a string of white-space/comma delimited names - (values will start at `start`), or an iterator/mapping of name, value pairs. + `names` should be either a string of white-space/comma delimited + names (values will start at `start`), or an iterator/mapping of + name, value pairs. `module` should be set to the module this class is being created in; - if it is not set, an attempt to find that module will be made, but if - it fails the class will not be picklable. + if it is not set, an attempt to find that module will be made, but + if it fails the class will not be picklable. - `qualname` should be set to the actual location this class can be found - at in its module; by default it is set to the global scope. If this is - not correct, unpickling will fail in some circumstances. + `qualname` should be set to the actual location this class can be + found at in its module; by default it is set to the global scope. + If this is not correct, unpickling will fail in some circumstances. `type`, if set, will be mixed in as the first base class. """ @@ -783,6 +784,8 @@ def __dir__(cls): '__class__', '__contains__', '__doc__', '__getitem__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__', + # Supported sunder names of Enum class + '_generate_next_value_', '_missing_', ] + members ) @@ -819,8 +822,8 @@ def __members__(cls): """ Returns a mapping of member name->value. - This mapping lists all enum members, including aliases. Note that this - is a read-only view of the internal mapping. + This mapping lists all enum members, including aliases. Note that + this is a read-only view of the internal mapping. """ return MappingProxyType(cls._member_map_) @@ -1296,7 +1299,8 @@ def __dir__(self): """ Returns public methods and other interesting attributes. """ - interesting = set() + # Initialize with supported sunder names + interesting = set(('_generate_next_value_', '_missing_', '_add_alias_', '_add_value_alias_')) if self.__class__._member_type_ is not object: interesting = set(object.__dir__(self)) for name in getattr(self, '__dict__', []): @@ -1616,10 +1620,14 @@ def __or__(self, other): if other_value is NotImplemented: return NotImplemented - for flag in self, other: - if self._get_value(flag) is None: - raise TypeError(f"'{flag}' cannot be combined with other flags with |") value = self._value_ + # _get_value(self) is self._value_ and _get_value(other) is + # other_value, so only walk the operands (to raise on the offending + # flag) when one of those values is actually None. + if value is None or other_value is None: + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with |") return self.__class__(value | other_value) def __and__(self, other): @@ -1627,10 +1635,11 @@ def __and__(self, other): if other_value is NotImplemented: return NotImplemented - for flag in self, other: - if self._get_value(flag) is None: - raise TypeError(f"'{flag}' cannot be combined with other flags with &") value = self._value_ + if value is None or other_value is None: + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with &") return self.__class__(value & other_value) def __xor__(self, other): @@ -1638,10 +1647,11 @@ def __xor__(self, other): if other_value is NotImplemented: return NotImplemented - for flag in self, other: - if self._get_value(flag) is None: - raise TypeError(f"'{flag}' cannot be combined with other flags with ^") value = self._value_ + if value is None or other_value is None: + for flag in self, other: + if self._get_value(flag) is None: + raise TypeError(f"'{flag}' cannot be combined with other flags with ^") return self.__class__(value ^ other_value) def __invert__(self): diff --git a/Lib/fractions.py b/Lib/fractions.py index c1b12e7a1c091c6..84e844cdd697687 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -339,24 +339,24 @@ def from_float(cls, f): Beware that Fraction.from_float(0.3) != Fraction(3, 10). """ + if isinstance(f, float): + return cls._from_coprime_ints(*f.as_integer_ratio()) if isinstance(f, numbers.Integral): return cls(f) - elif not isinstance(f, float): - raise TypeError("%s.from_float() only takes floats, not %r (%s)" % - (cls.__name__, f, type(f).__name__)) - return cls._from_coprime_ints(*f.as_integer_ratio()) + raise TypeError("%s.from_float() only takes floats, not %r (%s)" % + (cls.__name__, f, type(f).__name__)) @classmethod def from_decimal(cls, dec): """Converts a finite Decimal instance to a rational number, exactly.""" from decimal import Decimal + if isinstance(dec, Decimal): + return cls._from_coprime_ints(*dec.as_integer_ratio()) if isinstance(dec, numbers.Integral): - dec = Decimal(int(dec)) - elif not isinstance(dec, Decimal): - raise TypeError( - "%s.from_decimal() only takes Decimals, not %r (%s)" % - (cls.__name__, dec, type(dec).__name__)) - return cls._from_coprime_ints(*dec.as_integer_ratio()) + dec = int(dec) + return cls._from_coprime_ints(*dec.as_integer_ratio()) + raise TypeError("%s.from_decimal() only takes Decimals, not %r (%s)" % + (cls.__name__, dec, type(dec).__name__)) @classmethod def _from_coprime_ints(cls, numerator, denominator, /): diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 640acc64f620cc9..2f092d50f31782b 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -883,7 +883,16 @@ def ftpcp(source, sourcename, target, targetname = '', type = 'I'): type = 'TYPE ' + type source.voidcmd(type) target.voidcmd(type) - sourcehost, sourceport = parse227(source.sendcmd('PASV')) + # Don't trust the IPv4 address the source server advertises in its PASV + # reply: a malicious source could otherwise point the target's data + # connection at an arbitrary host (SSRF). A caller that needs the old + # behavior can set trust_server_pasv_ipv4_address on the source FTP + # object. See FTP.makepasv(), which applies the same rule. + untrusted_host, sourceport = parse227(source.sendcmd('PASV')) + if source.trust_server_pasv_ipv4_address: + sourcehost = untrusted_host + else: + sourcehost = source.sock.getpeername()[0] target.sendport(sourcehost, sourceport) # RFC 959: the user must "listen" [...] BEFORE sending the # transfer request. diff --git a/Lib/functools.py b/Lib/functools.py index cd374631f167925..8425f6030010f3d 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -232,9 +232,9 @@ def __ge__(self, other): ### reduce() sequence to a single item ################################################################################ -_initial_missing = object() +_initial_missing = sentinel('_initial_missing') -def reduce(function, sequence, initial=_initial_missing): +def reduce(function, sequence, /, initial=_initial_missing): """ reduce(function, iterable, /[, initial]) -> value @@ -264,6 +264,11 @@ def reduce(function, sequence, initial=_initial_missing): return value +try: + from _functools import reduce +except ImportError: + pass + ################################################################################ ### partial() argument application @@ -558,16 +563,16 @@ def lru_cache(maxsize=128, typed=False): If *maxsize* is set to None, the LRU features are disabled and the cache can grow without bound. - If *typed* is True, arguments of different types will be cached separately. - For example, f(decimal.Decimal("3.0")) and f(3.0) will be treated as - distinct calls with distinct results. Some types such as str and int may - be cached separately even when typed is false. + If *typed* is True, arguments of different types will be cached + separately. For example, f(decimal.Decimal("3.0")) and f(3.0) will be + treated as distinct calls with distinct results. Some types such as + str and int may be cached separately even when typed is false. Arguments to the cached function must be hashable. View the cache statistics named tuple (hits, misses, maxsize, currsize) - with f.cache_info(). Clear the cache and statistics with f.cache_clear(). - Access the underlying function with f.__wrapped__. + with f.cache_info(). Clear the cache and statistics with + f.cache_clear(). Access the underlying function with f.__wrapped__. See: https://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU) @@ -1178,31 +1183,3 @@ def __get__(self, instance, owner=None): return val __class_getitem__ = classmethod(GenericAlias) - -def _warn_python_reduce_kwargs(py_reduce): - @wraps(py_reduce) - def wrapper(*args, **kwargs): - if 'function' in kwargs or 'sequence' in kwargs: - import os - import warnings - warnings.warn( - 'Calling functools.reduce with keyword arguments ' - '"function" or "sequence" ' - 'is deprecated in Python 3.14 and will be ' - 'forbidden in Python 3.16.', - DeprecationWarning, - skip_file_prefixes=(os.path.dirname(__file__),)) - return py_reduce(*args, **kwargs) - return wrapper - -reduce = _warn_python_reduce_kwargs(reduce) -del _warn_python_reduce_kwargs - -# The import of the C accelerated version of reduce() has been moved -# here due to gh-121676. In Python 3.16, _warn_python_reduce_kwargs() -# should be removed and the import block should be moved back right -# after the definition of reduce(). -try: - from _functools import reduce -except ImportError: - pass diff --git a/Lib/glob.py b/Lib/glob.py index 575e4bcba5be0d5..5a8ff46137ba5e4 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -25,17 +25,17 @@ def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, The order of the returned list is undefined. Sort it if you need a particular order. - If `root_dir` is not None, it should be a path-like object specifying the - root directory for searching. It has the same effect as changing the - current directory before calling it (without actually - changing it). If pathname is relative, the result will contain - paths relative to `root_dir`. + If `root_dir` is not None, it should be a path-like object specifying + the root directory for searching. It has the same effect as changing + the current directory before calling it (without actually changing it). + If pathname is relative, the result will contain paths relative to + `root_dir`. If `dir_fd` is not None, it should be a file descriptor referring to a directory, and paths will then be relative to that directory. - If `include_hidden` is true, the patterns '*', '?', '**' will match hidden - directories. + If `include_hidden` is true, the patterns '*', '?', '**' will match + hidden directories. If `recursive` is true, the pattern '**' will match any files and zero or more directories and subdirectories. @@ -56,16 +56,16 @@ def iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, particular order. If `root_dir` is not None, it should be a path-like object specifying - the root directory for searching. It has the same effect as changing - the current directory before calling it (without actually - changing it). If pathname is relative, the result will contain - paths relative to `root_dir`. + the root directory for searching. It has the same effect as changing + the current directory before calling it (without actually changing it). + If pathname is relative, the result will contain paths relative to + `root_dir`. If `dir_fd` is not None, it should be a file descriptor referring to a directory, and paths will then be relative to that directory. - If `include_hidden` is true, the patterns '*', '?', '**' will match hidden - directories. + If `include_hidden` is true, the patterns '*', '?', '**' will match + hidden directories. If `recursive` is true, the pattern '**' will match any files and zero or more directories and subdirectories. @@ -279,15 +279,15 @@ def escape(pathname): def translate(pat, *, recursive=False, include_hidden=False, seps=None): """Translate a pathname with shell wildcards to a regular expression. - If `recursive` is true, the pattern segment '**' will match any number of - path segments. + If `recursive` is true, the pattern segment '**' will match any number + of path segments. If `include_hidden` is true, wildcards can match path segments beginning with a dot ('.'). If a sequence of separator characters is given to `seps`, they will be - used to split the pattern into segments and match path separators. If not - given, os.path.sep and os.path.altsep (where available) are used. + used to split the pattern into segments and match path separators. If + not given, os.path.sep and os.path.altsep (where available) are used. """ if not seps: if os.path.altsep: diff --git a/Lib/gzip.py b/Lib/gzip.py index 8a2faf846bf8941..247efc01527bb14 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -31,19 +31,19 @@ def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF, - encoding=None, errors=None, newline=None): + encoding=None, errors=None, newline=None, *, mtime=None): """Open a gzip-compressed file in binary or text mode. - The filename argument can be an actual filename (a str or bytes object), or - an existing file object to read from or write to. + The filename argument can be an actual filename (a str or bytes object), + or an existing file object to read from or write to. - The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for - binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is - "rb", and the default compresslevel is 9. + The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" + for binary mode, or "rt", "wt", "xt" or "at" for text mode. The default + mode is "rb", and the default compresslevel is 9. - For binary mode, this function is equivalent to the GzipFile constructor: - GzipFile(filename, mode, compresslevel). In this case, the encoding, errors - and newline arguments must not be provided. + For binary mode, this function is equivalent to the GzipFile + constructor: GzipFile(filename, mode, compresslevel). In this case, + the encoding, errors and newline arguments must not be provided. For text mode, a GzipFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling @@ -63,9 +63,10 @@ def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF, gz_mode = mode.replace("t", "") if isinstance(filename, (str, bytes, os.PathLike)): - binary_file = GzipFile(filename, gz_mode, compresslevel) + binary_file = GzipFile(filename, gz_mode, compresslevel, mtime=mtime) elif hasattr(filename, "read") or hasattr(filename, "write"): - binary_file = GzipFile(None, gz_mode, compresslevel, filename) + binary_file = GzipFile(None, gz_mode, compresslevel, filename, + mtime=mtime) else: raise TypeError("filename must be a str or bytes object, or a file") @@ -148,8 +149,8 @@ class GzipFile(_streams.BaseStream): """The GzipFile class simulates most of the methods of a file object with the exception of the truncate() method. - This class only supports opening files in binary mode. If you need to open a - compressed file in text mode, use the gzip.open() function. + This class only supports opening files in binary mode. If you need to + open a compressed file in text mode, use the gzip.open() function. """ @@ -165,31 +166,34 @@ def __init__(self, filename=None, mode=None, non-trivial value. The new class instance is based on fileobj, which can be a regular - file, an io.BytesIO object, or any other object which simulates a file. - It defaults to None, in which case filename is opened to provide - a file object. + file, an io.BytesIO object, or any other object which simulates + a file. It defaults to None, in which case filename is opened to + provide a file object. When fileobj is not None, the filename argument is only used to be included in the gzip file header, which may include the original filename of the uncompressed file. It defaults to the filename of fileobj, if discernible; otherwise, it defaults to the empty string, - and in this case the original filename is not included in the header. - - The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', 'x', or - 'xb' depending on whether the file will be read or written. The default - is the mode of fileobj if discernible; otherwise, the default is 'rb'. - A mode of 'r' is equivalent to one of 'rb', and similarly for 'w' and - 'wb', 'a' and 'ab', and 'x' and 'xb'. - - The compresslevel argument is an integer from 0 to 9 controlling the - level of compression; 1 is fastest and produces the least compression, - and 9 is slowest and produces the most compression. 0 is no compression - at all. The default is 9. - - The optional mtime argument is the timestamp requested by gzip. The time - is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If mtime is omitted or None, the current time is used. Use mtime = 0 - to generate a compressed stream that does not depend on creation time. + and in this case the original filename is not included in the + header. + + The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', + 'x', or 'xb' depending on whether the file will be read or written. + The default is the mode of fileobj if discernible; otherwise, the + default is 'rb'. A mode of 'r' is equivalent to one of 'rb', and + similarly for 'w' and 'wb', 'a' and 'ab', and 'x' and 'xb'. + + The compresslevel argument is an integer from 0 to 9 controlling + the level of compression; 1 is fastest and produces the least + compression, and 9 is slowest and produces the most compression. + 0 is no compression at all. The default is 9. + + The optional mtime argument is the timestamp requested by gzip. + The time is in Unix format, i.e., seconds since 00:00:00 UTC, + January 1, 1970. Set mtime to 0 to generate a compressed stream + that does not depend on creation time. If mtime is omitted or None, + the current time is used. If the resulting mtime is outside the + range 0 to 2**32-1, then the value 0 is used instead. """ @@ -295,6 +299,8 @@ def _write_gzip_header(self, compresslevel): mtime = self._write_mtime if mtime is None: mtime = time.time() + if not 0 <= mtime < 2**32: + mtime = 0 write32u(self.fileobj, int(mtime)) if compresslevel == _COMPRESS_LEVEL_BEST: xfl = b'\002' @@ -484,40 +490,73 @@ def _read_exact(fp, n): return data +def _read_until_null(fp, crc=None): + '''Read until the first encountered null byte in fp. + If crc is not None, update and return the CRC. + ''' + if crc is None: + while True: + s = fp.read(1) + if not s or s == b'\000': + break + else: + while True: + s = fp.read(1) + crc = zlib.crc32(s, crc) + if not s or s == b'\000': + break + return crc + + def _read_gzip_header(fp): '''Read a gzip header from `fp` and progress to the end of the header. Returns last mtime if header was present or None otherwise. ''' magic = fp.read(2) - if magic == b'': + if not magic: return None if magic != b'\037\213': raise BadGzipFile('Not a gzipped file (%r)' % magic) - - (method, flag, last_mtime) = struct.unpack("' % (self.__class__.__name__, self.OutputString()) - def _js_output(self, attrs=None): """Internal implementation without deprecation warning.""" - import base64 + import urllib.parse # Print javascript output_string = self.OutputString(attrs) if _has_control_character(output_string): raise CookieError("Control characters are not allowed in cookies") # Base64-encode value to avoid template # injection in cookie values. - output_encoded = base64.b64encode(output_string.encode('utf-8')).decode("ascii") + output_encoded = urllib.parse.quote(output_string, safe='', encoding='utf-8') return """ """ % (output_encoded,) diff --git a/Lib/http/server.py b/Lib/http/server.py index 16ea7f3f93693f7..ebc85052aecb900 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1081,7 +1081,7 @@ def _main(args=None): import argparse import contextlib - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-b', '--bind', metavar='ADDRESS', help='bind to this address ' '(default: all interfaces)') diff --git a/Lib/idlelib/idle_test/template.py b/Lib/idlelib/idle_test/template.py index 725a55b9c47230c..6e98556458ec588 100644 --- a/Lib/idlelib/idle_test/template.py +++ b/Lib/idlelib/idle_test/template.py @@ -1,6 +1,5 @@ "Test , coverage %." -from idlelib import zzdummy import unittest from test.support import requires from tkinter import Tk diff --git a/Lib/idlelib/idle_test/test_help.py b/Lib/idlelib/idle_test/test_help.py index ebb02b5c0d8356a..ebbaceb2eb7bbfd 100644 --- a/Lib/idlelib/idle_test/test_help.py +++ b/Lib/idlelib/idle_test/test_help.py @@ -4,7 +4,6 @@ import unittest from test.support import requires requires('gui') -from os.path import abspath, dirname, join from tkinter import Tk diff --git a/Lib/idlelib/idle_test/test_iomenu.py b/Lib/idlelib/idle_test/test_iomenu.py index e0642cf0cabef04..80f72bdfe5ff0ef 100644 --- a/Lib/idlelib/idle_test/test_iomenu.py +++ b/Lib/idlelib/idle_test/test_iomenu.py @@ -23,11 +23,10 @@ def setUpClass(cls): cls.root = Tk() cls.root.withdraw() cls.editwin = EditorWindow(root=cls.root) - cls.io = iomenu.IOBinding(cls.editwin) + cls.io = cls.editwin.io @classmethod def tearDownClass(cls): - cls.io.close() cls.editwin._close() del cls.editwin cls.root.update_idletasks() diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 464126e2df06682..fc502f7fde17808 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -61,6 +61,7 @@ def set_filename_change_hook(self, hook): self.filename_change_hook = hook filename = None + file_timestamp = None dirname = None def set_filename(self, filename): @@ -127,6 +128,7 @@ def loadfile(self, filename): chars = f.read() fileencoding = f.encoding eol_convention = f.newlines + file_timestamp = self.getmtime(filename) converted = False except (UnicodeDecodeError, SyntaxError): # Wait for the editor window to appear @@ -142,6 +144,7 @@ def loadfile(self, filename): chars = f.read() fileencoding = f.encoding eol_convention = f.newlines + file_timestamp = self.getmtime(filename) converted = True except OSError as err: messagebox.showerror("I/O Error", str(err), parent=self.text) @@ -170,6 +173,7 @@ def loadfile(self, filename): self.text.insert("1.0", chars) self.reset_undo() self.set_filename(filename) + self.file_timestamp = file_timestamp if converted: # We need to save the conversion results first # before being able to execute the code @@ -206,7 +210,26 @@ def save(self, event): if not self.filename: self.save_as(event) else: + # Check the time of most recent content modification so the + # user doesn't accidentally overwrite a newer version of the file. + try: + file_timestamp = self.getmtime(self.filename) + except OSError: + pass + else: + if self.file_timestamp != file_timestamp: + confirm = messagebox.askokcancel( + title="File has changed", + message=( + "The file has changed on disk since reading it!\n\n" + "Do you really want to overwrite it?"), + default=messagebox.CANCEL, + parent=self.text) + if not confirm: + return "break" + if self.writefile(self.filename): + self.file_timestamp = self.getmtime(self.filename) self.set_saved(True) try: self.editwin.store_file_breaks() @@ -219,6 +242,7 @@ def save_as(self, event): filename = self.asksavefile() if filename: if self.writefile(filename): + self.file_timestamp = self.getmtime(filename) self.set_filename(filename) self.set_saved(1) try: @@ -251,6 +275,9 @@ def writefile(self, filename): parent=self.text) return False + def getmtime(self, filename): + return os.stat(filename).st_mtime + def fixnewlines(self): """Return text with os eols. diff --git a/Lib/imaplib.py b/Lib/imaplib.py index cb3edceae0d9f19..497b5a60cecb083 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -313,25 +313,34 @@ def open(self, host='', port=IMAP4_PORT, timeout=None): self.host = host self.port = port self.sock = self._create_socket(timeout) - self._file = self.sock.makefile('rb') - + # Since IMAP4 implements its own read() and readline() buffering, + # the '_imaplib_file' attribute is unused. Nonetheless it is kept + # and exposed solely for backward compatibility purposes. + self._imaplib_file = self.sock.makefile('rb') @property def file(self): - # The old 'file' attribute is no longer used now that we do our own - # read() and readline() buffering, with which it conflicts. - # As an undocumented interface, it should never have been accessed by - # external code, and therefore does not warrant deprecation. - # Nevertheless, we provide this property for now, to avoid suddenly - # breaking any code in the wild that might have been using it in a - # harmless way. import warnings - warnings.warn( - 'IMAP4.file is unsupported, can cause errors, and may be removed.', - RuntimeWarning, - stacklevel=2) - return self._file + warnings._deprecated("IMAP4.file", remove=(3, 19)) + return self._imaplib_file + @file.setter + def file(self, value): + import warnings + warnings._deprecated("IMAP4.file", remove=(3, 19)) + # Ideally, we would want to close the previous file, + # but since we do not know how subclasses will use + # that setter, it is probably better to leave it to + # the caller. + self._imaplib_file = value + + def _close_imaplib_file(self): + file = self._imaplib_file + if file is not None: + try: + file.close() + except OSError: + pass def read(self, size): """Read 'size' bytes from remote.""" @@ -417,7 +426,7 @@ def send(self, data): def shutdown(self): """Close I/O established in "open".""" - self._file.close() + self._close_imaplib_file() try: self.sock.shutdown(socket.SHUT_RDWR) except OSError as exc: @@ -703,7 +712,7 @@ def login(self, user, password): """ typ, dat = self._simple_command('LOGIN', user, self._quote(password)) if typ != 'OK': - raise self.error(dat[-1]) + raise self.error(dat[-1].decode('UTF-8', 'replace')) self.state = 'AUTH' return typ, dat @@ -921,9 +930,10 @@ def starttls(self, ssl_context=None): ssl_context = ssl._create_stdlib_context() typ, dat = self._simple_command(name) if typ == 'OK': + self._close_imaplib_file() self.sock = ssl_context.wrap_socket(self.sock, server_hostname=self.host) - self._file = self.sock.makefile('rb') + self._imaplib_file = self.sock.makefile('rb') self._tls_established = True self._get_capabilities() else: @@ -1680,7 +1690,7 @@ def open(self, host=None, port=None, timeout=None): self.host = None # For compatibility with parent class self.port = None self.sock = None - self._file = None + self._imaplib_file = None self.process = subprocess.Popen(self.command, bufsize=DEFAULT_BUFFER_SIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 06dc45d71d7ca4a..eb1686a5c8217c6 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -20,12 +20,6 @@ # reference any injected objects! This includes not only global code but also # anything specified at the class level. -def _object_name(obj): - try: - return obj.__qualname__ - except AttributeError: - return type(obj).__qualname__ - # Bootstrap-related code ###################################################### # Modules injected manually by _setup() diff --git a/Lib/inspect.py b/Lib/inspect.py index af304f186a69bc2..af6aa3eb37a53bb 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1,7 +1,7 @@ """Get useful information from live Python objects. This module encapsulates the interface provided by the internal special -attributes (co_*, im_*, tb_*, etc.) in a friendlier fashion. +attributes (co_*, tb_*, etc.) in a friendlier fashion. It also provides some help for examining source code and class layout. Here are some of the useful functions provided by this module: @@ -1254,11 +1254,10 @@ def getargs(co): FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') -def getfullargspec(func): +def getfullargspec(func, *, annotation_format=Format.VALUE): """Get the names and default values of a callable object's parameters. - A tuple of seven things is returned: - (args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations). + A FullArgSpec namedtuple is returned, which has the following attributes: 'args' is a list of the parameter names. 'varargs' and 'varkw' are the names of the * and ** parameters or None. 'defaults' is an n-tuple of the default values of the last n parameters. @@ -1266,6 +1265,9 @@ def getfullargspec(func): 'kwonlydefaults' is a dictionary mapping names from kwonlyargs to defaults. 'annotations' is a dictionary mapping parameter names to annotations. + The *annotation_format* parameter controls the format of the annotations. + See the annotationlib documentation for details. + Notable differences from inspect.signature(): - the "self" parameter is always reported, even for bound methods - wrapper chains defined by __wrapped__ *not* unwrapped automatically @@ -1291,7 +1293,8 @@ def getfullargspec(func): follow_wrapper_chains=False, skip_bound_arg=False, sigcls=Signature, - eval_str=False) + eval_str=False, + annotation_format=annotation_format) except Exception as ex: # Most of the times 'signature' will raise ValueError. # But, it can also raise AttributeError, and, maybe something @@ -1706,9 +1709,13 @@ def _check_instance(obj, attr): def _check_class(klass, attr): + last_meta = None for entry in _static_getmro(klass): - if _shadowed_dict(type(entry)) is _sentinel and attr in entry.__dict__: - return entry.__dict__[attr] + meta = type(entry) + if meta is last_meta or _shadowed_dict(meta) is _sentinel: + last_meta = meta + if attr in entry.__dict__: + return entry.__dict__[attr] return _sentinel @@ -1740,6 +1747,9 @@ def _shadowed_dict(klass): # destroyed, and the dynamically created classes happen to be the only # objects that hold strong references to other objects that take up a # significant amount of memory. + # Fast path: `type` is the dominant caller; result is always _sentinel. + if klass is type: + return _sentinel return _shadowed_dict_from_weakref_mro_tuple( *[make_weakref(entry) for entry in _static_getmro(klass)] ) @@ -2197,7 +2207,8 @@ def wrap_value(s): except NameError: raise ValueError - if isinstance(value, (str, int, float, bytes, bool, type(None))): + if isinstance(value, (str, int, float, bytes, bool, type(None), + sentinel)): return ast.Constant(value) raise ValueError @@ -3445,11 +3456,11 @@ def _main(): import argparse import importlib - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument( 'object', help="The object to be analysed. " - "It supports the 'module:qualname' syntax") + "It supports the `module:qualname` syntax") parser.add_argument( '-d', '--details', action='store_true', help='Display info about the module rather than its source code') diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 251025efac14b87..37a86831ff94838 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -142,8 +142,8 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is @@ -206,8 +206,8 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. If specified, ``separators`` should be an ``(item_separator, key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is @@ -292,12 +292,13 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. - ``array_hook`` is an optional function that will be called with the result - of any literal array decode (a ``list``). The return value of this function will - be used instead of the ``list``. This feature can be used along - ``object_pairs_hook`` to customize the resulting data structure - for example, - by setting that to ``frozendict`` and ``array_hook`` to ``tuple``, one can get - a deep immutable data structute from any JSON data. + ``array_hook`` is an optional function that will be called with the + result of any literal array decode (a ``list``). The return value of + this function will be used instead of the ``list``. This feature can + be used along ``object_pairs_hook`` to customize the resulting data + structure - for example, by setting that to ``frozendict`` and + ``array_hook`` to ``tuple``, one can get a deep immutable data structure + from any JSON data. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. @@ -306,7 +307,7 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, cls=cls, object_hook=object_hook, parse_float=parse_float, parse_int=parse_int, parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, - array_hook=None, **kw) + array_hook=array_hook, **kw) def loads(s, *, cls=None, object_hook=None, parse_float=None, @@ -327,12 +328,13 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. - ``array_hook`` is an optional function that will be called with the result - of any literal array decode (a ``list``). The return value of this function will - be used instead of the ``list``. This feature can be used along - ``object_pairs_hook`` to customize the resulting data structure - for example, - by setting that to ``frozendict`` and ``array_hook`` to ``tuple``, one can get - a deep immutable data structute from any JSON data. + ``array_hook`` is an optional function that will be called with the + result of any literal array decode (a ``list``). The return value of + this function will be used instead of the ``list``. This feature can + be used along ``object_pairs_hook`` to customize the resulting data + structure - for example, by setting that to ``frozendict`` and + ``array_hook`` to ``tuple``, one can get a deep immutable data structure + from any JSON data. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to diff --git a/Lib/json/tool.py b/Lib/json/tool.py index e56a601c581ae5e..6385d971f733041 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -44,13 +44,13 @@ def _replace_match_callback(match): def main(): description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) parser.add_argument('infile', nargs='?', help='a JSON file to be validated or pretty-printed; ' - 'defaults to stdin', + 'defaults to `stdin`', default='-') parser.add_argument('outfile', nargs='?', - help='write the output of infile to outfile', + help='write the output of `infile` to `outfile`', default=None) parser.add_argument('--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key') @@ -58,7 +58,7 @@ def main(): help='disable escaping of non-ASCII characters') parser.add_argument('--json-lines', action='store_true', default=False, help='parse input using the JSON Lines format. ' - 'Use with --no-indent or --compact to produce valid JSON Lines output.') + 'Use with `--no-indent` or `--compact` to produce valid JSON Lines output.') group = parser.add_mutually_exclusive_group() group.add_argument('--indent', default=4, type=int, help='separate items with newlines and use this number ' diff --git a/Lib/locale.py b/Lib/locale.py index 4ff6f8c0f0a7754..25efff5b8568543 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -17,17 +17,14 @@ from builtins import str as _builtin_str import functools -# Try importing the _locale module. -# -# If this fails, fall back on a basic 'C' locale emulation. - # Yuck: LC_MESSAGES is non-standard: can't tell whether it exists before # trying the import. So __all__ is also fiddled at the end of the file. __all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error", "setlocale", "localeconv", "strcoll", "strxfrm", "str", "atof", "atoi", "format_string", "currency", "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY", - "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding"] + "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding", "delocalize", + "localize"] def _strcoll(a,b): """ strcoll(string,string) -> int. @@ -41,6 +38,9 @@ def _strxfrm(s): """ return s +# Try importing the _locale module. +# +# If this fails, fall back on a basic 'C' locale emulation. try: from _locale import * @@ -91,6 +91,29 @@ def setlocale(category, value=None): raise Error('_locale emulation only supports "C" locale') return 'C' +else: + _conditional_constants_names = ["ABDAY_1", "ABDAY_2", "ABDAY_3", + "ABDAY_4", "ABDAY_5", "ABDAY_6", + "ABDAY_7", "ABMON_1", "ABMON_10", + "ABMON_11", "ABMON_12", "ABMON_2", + "ABMON_3", "ABMON_4", "ABMON_5", + "ABMON_6", "ABMON_7", "ABMON_8", + "ABMON_9", "ALT_DIGITS", "CODESET", + "CRNCYSTR", "DAY_1", "DAY_2", "DAY_3", + "DAY_4", "DAY_5", "DAY_6", + "DAY_7", "D_FMT", "D_T_FMT", + "ERA", "ERA_D_FMT", "ERA_D_T_FMT", + "ERA_T_FMT", "MON_1", "MON_10", + "MON_11", "MON_12", "MON_2", "MON_3", + "MON_4", "MON_5", "MON_6", "MON_7", + "MON_8", "MON_9", "NOEXPR", + "RADIXCHAR", "THOUSEP", "T_FMT", + "T_FMT_AMPM", "YESEXPR", "AM_STR", + "PM_STR"] + # The constants defined in _locale are platform-dependent, + # so we only include those that are available on the current platform. + __all__.extend(vars().keys() & _conditional_constants_names) + # These may or may not exist in _locale, so be sure to set them. if 'strxfrm' not in globals(): strxfrm = _strxfrm diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 6eef90ae5cd9143..b4a5f5cc2f598f5 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1373,9 +1373,17 @@ def getLogger(self, name): logger and fix up the parent/child references which pointed to the placeholder to now point to the logger. """ - rv = None if not isinstance(name, str): raise TypeError('A logger name must be a string') + # Fast path: an already-registered, non-placeholder logger can be + # returned without taking the lock. dict.get() is atomic under both + # the GIL and free threading. A Logger is inserted into loggerDict only + # after it is fully wired up (parent/child references fixed) under the + # lock, so the fast path never observes a logger whose parent is not yet + # set. + rv = self.loggerDict.get(name) + if rv is not None and not isinstance(rv, PlaceHolder): + return rv with _lock: if name in self.loggerDict: rv = self.loggerDict[name] @@ -1383,14 +1391,18 @@ def getLogger(self, name): ph = rv rv = (self.loggerClass or _loggerClass)(name) rv.manager = self - self.loggerDict[name] = rv self._fixupChildren(ph, rv) self._fixupParents(rv) + # Publish only after rv is fully wired: the fast path reads + # loggerDict without the lock. + self.loggerDict[name] = rv else: rv = (self.loggerClass or _loggerClass)(name) rv.manager = self - self.loggerDict[name] = rv self._fixupParents(rv) + # Publish only after rv is fully wired: the fast path reads + # loggerDict without the lock. + self.loggerDict[name] = rv return rv def setLoggerClass(self, klass): diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 3d9aa00fa52d116..35233e731eb42d1 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -21,7 +21,7 @@ Copyright (C) 2001-2022 Vinay Sajip. All Rights Reserved. -To use, simply 'import logging' and log away! +To use, simply 'import logging.config' and log away! """ import errno @@ -36,6 +36,7 @@ import threading import traceback +from bisect import bisect_left from socketserver import ThreadingTCPServer, StreamRequestHandler @@ -186,9 +187,8 @@ def _handle_existing_loggers(existing, child_loggers, disable_existing): what was intended by the user. Also, allow existing loggers to NOT be disabled if disable_existing is false. """ - root = logging.root for log in existing: - logger = root.manager.loggerDict[log] + logger = logging.root.manager.loggerDict[log] if log in child_loggers: if not isinstance(logger, logging.PlaceHolder): logger.setLevel(logging.NOTSET) @@ -197,6 +197,20 @@ def _handle_existing_loggers(existing, child_loggers, disable_existing): else: logger.disabled = disable_existing +def _forget_existing_logger(name, existing, existing_set, child_loggers): + """Forget a configured logger and record its existing children.""" + prefixed = name + "." + i = bisect_left(existing, prefixed) + num_existing = len(existing) + while i < num_existing: + child = existing[i] + if not child.startswith(prefixed): + break + if child in existing_set: + child_loggers[child] = None + i += 1 + existing_set.remove(name) + def _install_loggers(cp, handlers, disable_existing): """Create and install loggers""" @@ -235,25 +249,18 @@ def _install_loggers(cp, handlers, disable_existing): #named loggers. With a sorted list it is easier #to find the child loggers. existing.sort() + existing_set = set(existing) #We'll keep the list of existing loggers #which are children of named loggers here... - child_loggers = [] + child_loggers = {} #now set up the new ones... for log in llist: section = cp["logger_%s" % log] qn = section["qualname"] propagate = section.getint("propagate", fallback=1) logger = logging.getLogger(qn) - if qn in existing: - i = existing.index(qn) + 1 # start with the entry after qn - prefixed = qn + "." - pflen = len(prefixed) - num_existing = len(existing) - while i < num_existing: - if existing[i][:pflen] == prefixed: - child_loggers.append(existing[i]) - i += 1 - existing.remove(qn) + if qn in existing_set: + _forget_existing_logger(qn, existing, existing_set, child_loggers) if "level" in section: level = section["level"] logger.setLevel(level) @@ -281,6 +288,7 @@ def _install_loggers(cp, handlers, disable_existing): # logger.propagate = 1 # elif disable_existing_loggers: # logger.disabled = 1 + existing = [name for name in existing if name in existing_set] _handle_existing_loggers(existing, child_loggers, disable_existing) @@ -638,22 +646,16 @@ def configure(self): #named loggers. With a sorted list it is easier #to find the child loggers. existing.sort() + existing_set = set(existing) #We'll keep the list of existing loggers #which are children of named loggers here... - child_loggers = [] + child_loggers = {} #now set up the new ones... loggers = config.get('loggers', EMPTY_DICT) for name in loggers: - if name in existing: - i = existing.index(name) + 1 # look after name - prefixed = name + "." - pflen = len(prefixed) - num_existing = len(existing) - while i < num_existing: - if existing[i][:pflen] == prefixed: - child_loggers.append(existing[i]) - i += 1 - existing.remove(name) + if name in existing_set: + _forget_existing_logger(name, existing, existing_set, + child_loggers) try: self.configure_logger(name, loggers[name]) except Exception as e: @@ -673,6 +675,7 @@ def configure(self): # logger.propagate = True # elif disable_existing: # logger.disabled = True + existing = [name for name in existing if name in existing_set] _handle_existing_loggers(existing, child_loggers, disable_existing) @@ -865,28 +868,7 @@ def configure_handler(self, config): else: factory = klass kwargs = {k: config[k] for k in config if (k != '.' and valid_ident(k))} - # When deprecation ends for using the 'strm' parameter, remove the - # "except TypeError ..." - try: - result = factory(**kwargs) - except TypeError as te: - if "'stream'" not in str(te): - raise - #The argument name changed from strm to stream - #Retry with old name. - #This is so that code can be used with older Python versions - #(e.g. by Django) - kwargs['strm'] = kwargs.pop('stream') - result = factory(**kwargs) - - import warnings - warnings.warn( - "Support for custom logging handlers with the 'strm' argument " - "is deprecated and scheduled for removal in Python 3.16. " - "Define handlers with the 'stream' argument instead.", - DeprecationWarning, - stacklevel=2, - ) + result = factory(**kwargs) if formatter: result.setFormatter(formatter) if level is not None: diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 575f2babbc47853..a5394d2dbea6494 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -282,7 +282,30 @@ def __init__(self, filename, when='h', interval=1, backupCount=0, # path object (see Issue #27493), but self.baseFilename will be a string filename = self.baseFilename if os.path.exists(filename): - t = int(os.stat(filename).st_mtime) + # Use the minimum of file creation and modification time as + # the base of the rollover calculation + creation_time = modification_time = None + if hasattr(os, 'statx'): + statx_result = os.statx(filename, + os.STATX_BTIME|os.STATX_CTIME|os.STATX_MTIME) + # Use stx_btime whenever it is available or use stx_ctime + # instead otherwise + creation_time = statx_result.stx_btime + if creation_time is None: + creation_time = statx_result.stx_ctime + modification_time = statx_result.stx_mtime + if creation_time is None or modification_time is None: + stat_result = os.stat(filename) + # Use st_birthtime whenever it is available or use st_ctime + # instead otherwise + if creation_time is None: + try: + creation_time = stat_result.st_birthtime + except AttributeError: + creation_time = stat_result.st_ctime + if modification_time is None: + modification_time = stat_result.st_mtime + t = int(min(creation_time, modification_time)) else: t = int(time.time()) self.rolloverAt = self.computeRollover(t) diff --git a/Lib/lzma.py b/Lib/lzma.py index 316066d024ea02f..fb4bbf650f849a4 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -13,6 +13,7 @@ "CHECK_ID_MAX", "CHECK_UNKNOWN", "FILTER_LZMA1", "FILTER_LZMA2", "FILTER_DELTA", "FILTER_X86", "FILTER_IA64", "FILTER_ARM", "FILTER_ARMTHUMB", "FILTER_POWERPC", "FILTER_SPARC", + "FILTER_ARM64", "FILTER_RISCV", "FORMAT_AUTO", "FORMAT_XZ", "FORMAT_ALONE", "FORMAT_RAW", "MF_HC3", "MF_HC4", "MF_BT2", "MF_BT3", "MF_BT4", "MODE_FAST", "MODE_NORMAL", "PRESET_DEFAULT", "PRESET_EXTREME", diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index a834826114614dd..4339ef5a61397dd 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -86,6 +86,9 @@ def add_type(self, type, ext, strict=True): is already known the extension will be added to the list of known extensions. + Registered lower-case extensions are matched + case-insensitively. + If strict is true, information will be added to list of standard types, else to the list of non-standard types. @@ -93,14 +96,7 @@ def add_type(self, type, ext, strict=True): Valid extensions are empty or start with a '.'. """ if ext and not ext.startswith('.'): - from warnings import _deprecated - - _deprecated( - "Undotted extensions", - "Using undotted extensions is deprecated and " - "will raise a ValueError in Python {remove}", - remove=(3, 16), - ) + raise ValueError(f"Extension {ext!r} must start with '.'") if not type: return @@ -179,23 +175,33 @@ def guess_file_type(self, path, *, strict=True): def _guess_file_type(self, path, strict, splitext): base, ext = splitext(path) - while (ext_lower := ext.lower()) in self.suffix_map: - base, ext = splitext(base + self.suffix_map[ext_lower]) + while True: + if ext in self.suffix_map: + suffix = self.suffix_map[ext] + elif (ext_lower := ext.lower()) in self.suffix_map: + suffix = self.suffix_map[ext_lower] + else: + break + base, ext = splitext(base + suffix) # encodings_map is case sensitive if ext in self.encodings_map: encoding = self.encodings_map[ext] base, ext = splitext(base) else: encoding = None - ext = ext.lower() + ext_lower = ext.lower() types_map = self.types_map[True] if ext in types_map: return types_map[ext], encoding + if ext_lower in types_map: + return types_map[ext_lower], encoding elif strict: return None, encoding types_map = self.types_map[False] if ext in types_map: return types_map[ext], encoding + if ext_lower in types_map: + return types_map[ext_lower], encoding else: return None, encoding @@ -393,6 +399,9 @@ def add_type(type, ext, strict=True): is already known the extension will be added to the list of known extensions. + Registered lower-case extensions are matched + case-insensitively. + If strict is true, information will be added to list of standard types, else to the list of non-standard types. @@ -599,10 +608,14 @@ def _default_mime_types(): '.ra' : 'audio/x-pn-realaudio', '.wav' : 'audio/vnd.wave', '.weba' : 'audio/webm', + '.ttc' : 'font/collection', '.otf' : 'font/otf', '.ttf' : 'font/ttf', '.woff' : 'font/woff', '.woff2' : 'font/woff2', + '.hjif' : 'haptics/hjif', + '.hmpg' : 'haptics/hmpg', + '.ivs' : 'haptics/ivs', '.avif' : 'image/avif', '.bmp' : 'image/bmp', '.emf' : 'image/emf', @@ -708,7 +721,7 @@ def _parse_args(args): from argparse import ArgumentParser parser = ArgumentParser( - description='map filename extensions to MIME types', color=True + description='map filename extensions to MIME types', ) parser.add_argument( '-e', '--extension', diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index e37ec07d722ca86..be2fc3edf5dce66 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -50,7 +50,7 @@ default_family = 'AF_INET' families = ['AF_INET'] -if hasattr(socket, 'AF_UNIX'): +if hasattr(socket, 'AF_UNIX') and sys.platform != 'cygwin': default_family = 'AF_UNIX' families += ['AF_UNIX'] diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index e1d251456024c07..45c393798deaca2 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -326,8 +326,10 @@ def _check_available(self): _concrete_contexts = { 'fork': ForkContext(), 'spawn': SpawnContext(), - 'forkserver': ForkServerContext(), } + if reduction.HAVE_SEND_HANDLE: + _concrete_contexts['forkserver'] = ForkServerContext() + # bpo-33725: running arbitrary code after fork() is no longer reliable # on macOS since macOS 10.14 (Mojave). Use spawn by default instead. # gh-84559: We changed everyones default to a thread safeish one in 3.14. diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 7d637325240f1cb..811e796f7766e94 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -152,12 +152,14 @@ def splitdrive(p, /): It is always true that: result[0] + result[1] == p - If the path contained a drive letter, drive_or_unc will contain everything - up to and including the colon. e.g. splitdrive("c:/dir") returns ("c:", "/dir") + If the path contained a drive letter, drive_or_unc will contain + everything up to and including the colon. e.g. splitdrive("c:/dir") + returns ("c:", "/dir") - If the path contained a UNC path, the drive_or_unc will contain the host name - and share up to but not including the fourth directory separator character. - e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir") + If the path contained a UNC path, the drive_or_unc will contain the + host name and share up to but not including the fourth directory + separator character. e.g. splitdrive("//host/computer/dir") returns + ("//host/computer", "/dir") Paths cannot contain both a drive letter and a UNC path. @@ -222,8 +224,8 @@ def splitroot(p, /): def split(p, /): """Split a pathname. - Return tuple (head, tail) where tail is everything after the final slash. - Either part may be empty.""" + Return tuple (head, tail) where tail is everything after the final + slash. Either part may be empty.""" p = os.fspath(p) seps = _get_bothseps(p) d, r, p = splitroot(p) diff --git a/Lib/opcode.py b/Lib/opcode.py index 4e60fb5af34f220..750d83b2c87af58 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -44,7 +44,7 @@ builtins.set, # Append-only — must match CONSTANT_* in # Include/internal/pycore_opcode_utils.h. - None, "", True, False, -1] + None, "", True, False, -1, builtins.frozenset, ()] _nb_ops = _opcode.get_nb_ops() hascompare = [opmap["COMPARE_OP"]] diff --git a/Lib/os.py b/Lib/os.py index 52cbc5bc85864e7..a5e1d8055569988 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -219,14 +219,17 @@ def _add(str, fn): # Super directory utilities. # (Inspired by Eric Raymond; the doc strings are mostly his) -def makedirs(name, mode=0o777, exist_ok=False): - """makedirs(name [, mode=0o777][, exist_ok=False]) - - Super-mkdir; create a leaf directory and all intermediate ones. Works like - mkdir, except that any intermediate path segment (not just the rightmost) - will be created if it does not exist. If the target directory already - exists, raise an OSError if exist_ok is False. Otherwise no exception is - raised. This is recursive. +def makedirs(name, mode=0o777, exist_ok=False, *, parent_mode=None): + """makedirs(name [, mode=0o777][, exist_ok=False][, parent_mode=None]) + + Super-mkdir; create a leaf directory and all intermediate ones. Works + like mkdir, except that any intermediate path segment (not just the + rightmost) will be created if it does not exist. If the target + directory already exists, raise an OSError if exist_ok is False. + Otherwise no exception is raised. If parent_mode is not None, it will + be used as the mode for any newly-created, intermediate-level + directories. Otherwise, intermediate directories are created with the + default permissions (respecting umask). This is recursive. """ head, tail = path.split(name) @@ -234,7 +237,11 @@ def makedirs(name, mode=0o777, exist_ok=False): head, tail = path.split(head) if head and tail and not path.exists(head): try: - makedirs(head, exist_ok=exist_ok) + if parent_mode is not None: + makedirs(head, mode=parent_mode, exist_ok=exist_ok, + parent_mode=parent_mode) + else: + makedirs(head, exist_ok=exist_ok) except FileExistsError: # Defeats race condition when another thread created the path pass @@ -314,12 +321,12 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirpath, dirnames, filenames dirpath is a string, the path to the directory. dirnames is a list of - the names of the subdirectories in dirpath (including symlinks to directories, - and excluding '.' and '..'). + the names of the subdirectories in dirpath (including symlinks to + directories, and excluding '.' and '..'). filenames is a list of the names of the non-directory files in dirpath. - Note that the names in the lists are just names, with no path components. - To get a full path (which begins with top) to a file or directory in - dirpath, do os.path.join(dirpath, name). + Note that the names in the lists are just names, with no path + components. To get a full path (which begins with top) to a file or + directory in dirpath, do os.path.join(dirpath, name). If optional arg 'topdown' is true or not specified, the triple for a directory is generated before the triples for any of its subdirectories @@ -329,13 +336,13 @@ def walk(top, topdown=True, onerror=None, followlinks=False): When topdown is true, the caller can modify the dirnames list in-place (e.g., via del or slice assignment), and walk will only recurse into the - subdirectories whose names remain in dirnames; this can be used to prune the - search, or to impose a specific order of visiting. Modifying dirnames when - topdown is false has no effect on the behavior of os.walk(), since the - directories in dirnames have already been generated by the time dirnames - itself is generated. No matter the value of topdown, the list of - subdirectories is retrieved before the tuples for the directory and its - subdirectories are generated. + subdirectories whose names remain in dirnames; this can be used to prune + the search, or to impose a specific order of visiting. Modifying + dirnames when topdown is false has no effect on the behavior of + os.walk(), since the directories in dirnames have already been generated + by the time dirnames itself is generated. No matter the value of + topdown, the list of subdirectories is retrieved before the tuples for + the directory and its subdirectories are generated. By default errors from the os.scandir() call are ignored. If optional arg 'onerror' is specified, it should be a function; it @@ -462,9 +469,9 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= The advantage of fwalk() over walk() is that it's safe against symlink races (when follow_symlinks is False). - If dir_fd is not None, it should be a file descriptor open to a directory, - and top should be relative; top will then be relative to that directory. - (dir_fd is always supported for fwalk.) + If dir_fd is not None, it should be a file descriptor open to + a directory, and top should be relative; top will then be relative to + that directory. (dir_fd is always supported for fwalk.) Caution: Since fwalk() yields file descriptors, those are only valid until the diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 44f967eb12dd4fd..ffec9c545ee11f8 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -12,12 +12,13 @@ import posixpath import sys from errno import * -from glob import _StringGlobber, _no_recurse_symlinks from itertools import chain from stat import ( S_IMODE, S_ISDIR, S_ISREG, S_ISLNK, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO, ) from _collections_abc import Sequence +lazy import shutil +lazy from glob import _StringGlobber, _no_recurse_symlinks try: import pwd @@ -988,6 +989,7 @@ def read_text(self, encoding=None, errors=None, newline=None): def write_bytes(self, data): """ Open the file in bytes mode, write to it, and close the file. + Return the number of bytes written. """ # type-check for the buffer interface before truncating the file view = memoryview(data) @@ -997,6 +999,7 @@ def write_bytes(self, data): def write_text(self, data, encoding=None, errors=None, newline=None): """ Open the file in text mode, write to it, and close the file. + Return the number of characters written. """ # Call io.text_encoding() here to ensure any warning is raised at an # appropriate stack level. @@ -1201,7 +1204,7 @@ def touch(self, mode=0o666, exist_ok=True): fd = os.open(self, flags, mode) os.close(fd) - def mkdir(self, mode=0o777, parents=False, exist_ok=False): + def mkdir(self, mode=0o777, parents=False, exist_ok=False, *, parent_mode=None): """ Create a new directory at this given path. """ @@ -1210,7 +1213,11 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): except FileNotFoundError: if not parents or self.parent == self: raise - self.parent.mkdir(parents=True, exist_ok=True) + if parent_mode is not None: + self.parent.mkdir(mode=parent_mode, parents=True, exist_ok=True, + parent_mode=parent_mode) + else: + self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) except OSError: # Cannot rely on checking for EEXIST, since the operating system @@ -1255,8 +1262,6 @@ def _delete(self): if self.is_symlink() or self.is_junction(): self.unlink() elif self.is_dir(): - # Lazy import to improve module import time - import shutil shutil.rmtree(self) else: self.unlink() diff --git a/Lib/pathlib/types.py b/Lib/pathlib/types.py index f21ce0774548f87..bb4a521223da04d 100644 --- a/Lib/pathlib/types.py +++ b/Lib/pathlib/types.py @@ -431,6 +431,7 @@ def __open_writer__(self, mode): def write_bytes(self, data): """ Open the file in bytes mode, write to it, and close the file. + Return the number of bytes written. """ # type-check for the buffer interface before truncating the file view = memoryview(data) @@ -440,6 +441,7 @@ def write_bytes(self, data): def write_text(self, data, encoding=None, errors=None, newline=None): """ Open the file in text mode, write to it, and close the file. + Return the number of characters written. """ # Call io.text_encoding() here to ensure any warning is raised at an # appropriate stack level. diff --git a/Lib/pdb.py b/Lib/pdb.py index 569599d349c49e6..01451f0229cacb2 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -402,6 +402,7 @@ def __init__(self, pdb_instance, stdin, stdout, prompt): completer_delims=frozenset(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') ) ) + self.readline_wrapper.get_reader().gen_colors = self.gen_colors def readline(self): @@ -463,6 +464,26 @@ def complete(self, text, state): except IndexError: return None + def gen_colors(self, buffer): + from _pyrepl.utils import ColorSpan, Span + + if not buffer.strip(): + return + + leading_spaces = len(buffer) - len(buffer.lstrip()) + leading_text = buffer.split()[0] + if hasattr(self.pdb_instance, 'do_' + leading_text): + yield ColorSpan( + Span(leading_spaces, leading_spaces + len(leading_text) - 1), + "soft_keyword" + ) + # Redact the command text with spaces so there will be no duplicated + # color span generated for it later. + redact_length = leading_spaces + len(leading_text) + buffer = ' ' * redact_length + buffer[redact_length:] + + yield from _pyrepl.utils.gen_colors(buffer) + class Pdb(bdb.Bdb, cmd.Cmd): _previous_sigint_handler = None @@ -3757,13 +3778,12 @@ def parse_args(): description=_usage, formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False, - color=True, ) # Get all the commands out first. For backwards compatibility, we allow # -c commands to be after the target. parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', - help='pdb commands to execute as if given in a .pdbrc file') + help='pdb commands to execute as if given in a `.pdbrc` file') opts, args = parser.parse_known_args() @@ -3789,6 +3809,10 @@ def parse_args(): opt_module = parser.parse_args(args[:2]) opts.module = opt_module.module args = args[2:] + elif args[0] == '--': + args.pop(0) + if not args: + parser.error("missing script or module to run") elif args[0].startswith('-'): # Invalid argument before the script name. invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) diff --git a/Lib/pickle.py b/Lib/pickle.py index 95836afdc2b43e5..f92b1fde768fc7d 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1937,7 +1937,6 @@ def _main(args=None): import pprint parser = argparse.ArgumentParser( description='display contents of the pickle files', - color=True, ) parser.add_argument( 'pickle_file', diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 976e218db192980..a9711538dae342b 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -2899,7 +2899,7 @@ def _main(args=None): help='preserve memo between disassemblies') parser.add_argument( '-l', '--indentlevel', default=4, type=int, - help='the number of blanks by which to indent a new MARK level') + help='the number of blanks by which to indent a new `MARK` level') parser.add_argument( '-a', '--annotate', action='store_true', help='annotate each line with a short opcode description') diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 11c2a4b0ef46350..9121d6a1e2285ce 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -443,7 +443,7 @@ def resolve_name(name, *, strict=False): within the imported package to get to the desired object. """ global _LENIENT_PATTERN, _STRICT_PATTERN - dotted_words = r'(?!\d)(\w+)(\.(?!\d)(\w+))*' + dotted_words = r'([\w--\d]\w*)(\.([\w--\d]\w*))*' if strict: if _STRICT_PATTERN is None: _STRICT_PATTERN = re.compile( diff --git a/Lib/platform.py b/Lib/platform.py index 9d7aa5c66a91cb3..36489d4fdd98ae2 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1400,7 +1400,7 @@ def invalidate_caches(): def _parse_args(args: list[str] | None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument("args", nargs="*", choices=["nonaliased", "terse"]) parser.add_argument( "--terse", diff --git a/Lib/profiling/sampling/__init__.py b/Lib/profiling/sampling/__init__.py index 6a0bb5e5c2f3877..71579a3903253ed 100644 --- a/Lib/profiling/sampling/__init__.py +++ b/Lib/profiling/sampling/__init__.py @@ -9,6 +9,15 @@ from .stack_collector import CollapsedStackCollector from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector +from .jsonl_collector import JsonlCollector from .string_table import StringTable -__all__ = ("Collector", "PstatsCollector", "CollapsedStackCollector", "HeatmapCollector", "GeckoCollector", "StringTable") +__all__ = ( + "Collector", + "PstatsCollector", + "CollapsedStackCollector", + "HeatmapCollector", + "GeckoCollector", + "JsonlCollector", + "StringTable", +) diff --git a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js index 1611bf754424c13..840acf2c27d1201 100644 --- a/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js +++ b/Lib/profiling/sampling/_flamegraph_assets/flamegraph.js @@ -7,6 +7,7 @@ let invertedData = null; let currentThreadFilter = 'all'; let isInverted = false; let useModuleNames = true; +let zoomedNodeValue = null; // Heat colors are now defined in CSS variables (--heat-1 through --heat-8) // and automatically switch with theme changes - no JS color arrays needed! @@ -316,6 +317,7 @@ function createPythonTooltip(data) { const selfSamples = d.data.self || 0; const selfMs = (selfSamples / 1000).toFixed(2); const percentage = ((d.data.value / data.value) * 100).toFixed(2); + const relativePercentage = Math.min(100, ((d.data.value / (zoomedNodeValue ?? data.value)) * 100)).toFixed(2); const calls = d.data.calls || 0; const childCount = d.children ? d.children.length : 0; const source = d.data.source; @@ -439,6 +441,11 @@ function createPythonTooltip(data) { Percentage: ${percentage}% + ${relativePercentage != percentage && relativePercentage != "100.00" ? ` + Relative Percentage: + ${relativePercentage}% + ` : ''} + ${calls > 0 ? ` Function Calls: ${calls.toLocaleString()} @@ -620,6 +627,9 @@ function createFlamegraph(tooltip, rootValue, data) { const percentage = d.data.value / rootValue; const level = getHeatLevel(percentage); return heatColors[level]; + }) + .onClick(function (d) { + zoomedNodeValue = d.data.value; }); return chart; @@ -629,6 +639,7 @@ function renderFlamegraph(chart, data) { d3.select("#chart").datum(data).call(chart); window.flamegraphChart = chart; window.flamegraphData = data; + zoomedNodeValue = null; populateStats(data); } @@ -1269,6 +1280,7 @@ function filterDataByThread(data, threadId) { function resetZoom() { if (window.flamegraphChart) { + zoomedNodeValue = null; window.flamegraphChart.resetZoom(); } } diff --git a/Lib/profiling/sampling/_heatmap_assets/heatmap.js b/Lib/profiling/sampling/_heatmap_assets/heatmap.js index 2da1103b82a52a3..1f698779f3a46e3 100644 --- a/Lib/profiling/sampling/_heatmap_assets/heatmap.js +++ b/Lib/profiling/sampling/_heatmap_assets/heatmap.js @@ -84,7 +84,7 @@ function showNavigationMenu(button, items, title) { item.appendChild(funcDiv); item.appendChild(createElement('div', 'callee-menu-file', linkData.file)); - item.addEventListener('click', () => window.location.href = linkData.link); + item.addEventListener('click', () => navigateToLine(linkData.link)); menu.appendChild(item); }); @@ -105,7 +105,7 @@ function handleNavigationClick(button, e) { const navData = button.getAttribute('data-nav'); if (navData) { - window.location.href = JSON.parse(navData).link; + navigateToLine(JSON.parse(navData).link); return; } @@ -117,11 +117,29 @@ function handleNavigationClick(button, e) { } } +function restartLineHighlight(target) { + target.style.animation = 'none'; + // Force style recalculation so restoring the animation restarts it. + void target.offsetWidth; + target.style.animation = ''; +} + +function navigateToLine(link) { + const url = new URL(link, window.location.href); + + if (url.href === window.location.href) { + scrollToTargetLine(); + } else { + window.location.href = link; + } +} + function scrollToTargetLine() { if (!window.location.hash) return; const target = document.querySelector(window.location.hash); if (target) { target.scrollIntoView({ behavior: 'smooth', block: 'start' }); + restartLineHighlight(target); } } diff --git a/Lib/profiling/sampling/binary_reader.py b/Lib/profiling/sampling/binary_reader.py index a11be3652597a6b..a29dad91ae339d1 100644 --- a/Lib/profiling/sampling/binary_reader.py +++ b/Lib/profiling/sampling/binary_reader.py @@ -4,6 +4,7 @@ from .gecko_collector import GeckoCollector from .stack_collector import FlamegraphCollector, CollapsedStackCollector +from .jsonl_collector import JsonlCollector from .pstats_collector import PstatsCollector @@ -117,6 +118,8 @@ def convert_binary_to_format(input_file, output_file, output_format, collector = PstatsCollector(interval) elif output_format == 'gecko': collector = GeckoCollector(interval) + elif output_format == "jsonl": + collector = JsonlCollector(interval) else: raise ValueError(f"Unknown output format: {output_format}") diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index bc879c43e15965f..a5d9573ae6b6ddd 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -20,6 +20,7 @@ from .stack_collector import CollapsedStackCollector, FlamegraphCollector, DiffFlamegraphCollector from .heatmap_collector import HeatmapCollector from .gecko_collector import GeckoCollector +from .jsonl_collector import JsonlCollector from .binary_collector import BinaryCollector from .binary_reader import BinaryReader from .constants import ( @@ -101,6 +102,7 @@ def __call__(self, parser, namespace, values, option_string=None): "diff_flamegraph": "html", "gecko": "json", "heatmap": "html", + "jsonl": "jsonl", "binary": "bin", } @@ -111,6 +113,7 @@ def __call__(self, parser, namespace, values, option_string=None): "diff_flamegraph": DiffFlamegraphCollector, "gecko": GeckoCollector, "heatmap": HeatmapCollector, + "jsonl": JsonlCollector, "binary": BinaryCollector, } @@ -164,7 +167,9 @@ def _build_child_profiler_args(args): child_args.extend(["--mode", mode]) # Format options (skip pstats as it's the default) - if args.format != "pstats": + if args.format == "diff_flamegraph": + child_args.extend(["--diff-flamegraph", args.diff_baseline]) + elif args.format != "pstats": child_args.append(f"--{args.format}") return child_args @@ -389,13 +394,13 @@ def _add_sampling_options(parser): sampling_group.add_argument( "--native", action="store_true", - help='Include artificial "" frames to denote calls to non-Python code', + help='Include artificial `` frames to denote calls to non-Python code', ) sampling_group.add_argument( "--no-gc", action="store_false", dest="gc", - help='Don\'t include artificial "" frames to denote active garbage collection', + help='Don\'t include artificial `` frames to denote active garbage collection', ) sampling_group.add_argument( "--opcodes", @@ -432,14 +437,14 @@ def _add_mode_options(parser): help="Sampling mode: wall (all samples), cpu (only samples when thread is on CPU), " "gil (only samples when thread holds the GIL), " "exception (only samples when thread has an active exception). " - "Incompatible with --async-aware", + "Incompatible with `--async-aware`", ) mode_group.add_argument( "--async-mode", choices=["running", "all"], default="running", help='Async profiling mode: "running" (only running task) ' - 'or "all" (all tasks including waiting). Requires --async-aware', + 'or "all" (all tasks including waiting). Requires `--async-aware`', ) @@ -486,7 +491,14 @@ def _add_format_options(parser, include_compression=True, include_binary=True): "--diff-flamegraph", metavar="BASELINE", action=DiffFlamegraphAction, - help="Generate differential flamegraph comparing current profile to BASELINE binary file", + help="Generate differential flamegraph comparing current profile to `BASELINE` binary file", + ) + format_group.add_argument( + "--jsonl", + action="store_const", + const="jsonl", + dest="format", + help="Generate newline-delimited JSON (JSONL) for programmatic consumers", ) if include_binary: format_group.add_argument( @@ -494,7 +506,7 @@ def _add_format_options(parser, include_compression=True, include_binary=True): action="store_const", const="binary", dest="format", - help="Generate high-performance binary format (use 'replay' command to convert)", + help="Generate high-performance binary format (use `replay` command to convert)", ) parser.set_defaults(format="pstats", diff_baseline=None) @@ -510,14 +522,14 @@ def _add_format_options(parser, include_compression=True, include_binary=True): "-o", "--output", dest="outfile", - help="Output path (default: stdout for pstats text; with -o, pstats is binary). " - "Auto-generated for other formats. For heatmap: directory name (default: heatmap_PID)", + help="Output path (default: `stdout` for `pstats` text; with `-o`, `pstats` is binary). " + "Auto-generated for other formats. For heatmap: directory name (default: `heatmap_PID`)", ) output_group.add_argument( "--browser", action="store_true", help="Automatically open HTML output (flamegraph, heatmap) in browser. " - "When using --subprocesses, only the main process opens the browser", + "When using `--subprocesses`, only the main process opens the browser", ) @@ -564,13 +576,13 @@ def _add_dump_options(parser): dump_group.add_argument( "--native", action="store_true", - help='Include artificial "" frames to denote calls to non-Python code', + help='Include artificial `` frames to denote calls to non-Python code', ) dump_group.add_argument( "--no-gc", action="store_false", dest="gc", - help='Don\'t include artificial "" frames to denote active garbage collection', + help='Don\'t include artificial `` frames to denote active garbage collection', ) dump_group.add_argument( "--opcodes", @@ -588,7 +600,7 @@ def _add_dump_options(parser): default=argparse.SUPPRESS, help='Async stack mode: "running" (only running task) ' 'or "all" (all tasks including waiting, default for dump). ' - "Requires --async-aware", + "Requires `--async-aware`", ) dump_group.add_argument( "--blocking", @@ -611,15 +623,18 @@ def _sort_to_mode(sort_choice): return sort_map.get(sort_choice, SORT_MODE_NSAMPLES) def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=False, - output_file=None, compression='auto', diff_baseline=None): + mode=None, output_file=None, compression='auto', + diff_baseline=None): """Create the appropriate collector based on format type. Args: - format_type: The output format ('pstats', 'collapsed', 'flamegraph', 'gecko', 'heatmap', 'binary', 'diff_flamegraph') + format_type: The output format ('pstats', 'collapsed', 'flamegraph', + 'gecko', 'heatmap', 'jsonl', 'binary', 'diff_flamegraph') sample_interval_usec: Sampling interval in microseconds skip_idle: Whether to skip idle samples opcodes: Whether to collect opcode information (only used by gecko format for creating interval markers in Firefox Profiler) + mode: Profiling mode for collectors that expose it in metadata output_file: Output file path (required for binary format) compression: Compression type for binary format ('auto', 'zstd', 'none') diff_baseline: Path to baseline binary file for differential flamegraph @@ -655,6 +670,11 @@ def _create_collector(format_type, sample_interval_usec, skip_idle, opcodes=Fals skip_idle = False return collector_class(sample_interval_usec, skip_idle=skip_idle, opcodes=opcodes) + if format_type == "jsonl": + return collector_class( + sample_interval_usec, skip_idle=skip_idle, mode=mode + ) + return collector_class(sample_interval_usec, skip_idle=skip_idle) @@ -998,7 +1018,7 @@ def _main(): "-m", "--module", action="store_true", - help="Run target as a module (like python -m)", + help="Run target as a module (like `python -m`)", ) run_parser.add_argument( "target", @@ -1142,7 +1162,7 @@ def _handle_attach(args): # Create the appropriate collector collector = _create_collector( - args.format, args.sample_interval_usec, skip_idle, args.opcodes, + args.format, args.sample_interval_usec, skip_idle, args.opcodes, mode, output_file=output_file, compression=getattr(args, 'compression', 'auto'), diff_baseline=args.diff_baseline @@ -1249,7 +1269,7 @@ def _handle_run(args): # Create the appropriate collector collector = _create_collector( - args.format, args.sample_interval_usec, skip_idle, args.opcodes, + args.format, args.sample_interval_usec, skip_idle, args.opcodes, mode, output_file=output_file, compression=getattr(args, 'compression', 'auto'), diff_baseline=args.diff_baseline diff --git a/Lib/profiling/sampling/collector.py b/Lib/profiling/sampling/collector.py index 08759b611696b7d..8e0f0c44c4f8f36 100644 --- a/Lib/profiling/sampling/collector.py +++ b/Lib/profiling/sampling/collector.py @@ -20,13 +20,16 @@ def normalize_location(location): """Normalize location to a 4-tuple format. Args: - location: tuple (lineno, end_lineno, col_offset, end_col_offset) or None + location: tuple (lineno, end_lineno, col_offset, end_col_offset), + an integer line number, or None Returns: tuple: (lineno, end_lineno, col_offset, end_col_offset) """ if location is None: return DEFAULT_LOCATION + if isinstance(location, int): + return (location, location, -1, -1) return location @@ -34,13 +37,16 @@ def extract_lineno(location): """Extract lineno from location. Args: - location: tuple (lineno, end_lineno, col_offset, end_col_offset) or None + location: tuple (lineno, end_lineno, col_offset, end_col_offset), + an integer line number, or None Returns: int: The line number (0 for synthetic frames) """ if location is None: return 0 + if isinstance(location, int): + return location return location[0] def _is_internal_frame(frame): @@ -137,6 +143,8 @@ def iter_async_frames(awaited_info_list): class Collector(ABC): + aggregating = False + @abstractmethod def collect(self, stack_frames, timestamps_us=None): """Collect profiling data from stack frames. diff --git a/Lib/profiling/sampling/constants.py b/Lib/profiling/sampling/constants.py index a364d0b8fde1e04..d7c710f943b1b77 100644 --- a/Lib/profiling/sampling/constants.py +++ b/Lib/profiling/sampling/constants.py @@ -11,6 +11,14 @@ PROFILING_MODE_ALL = 3 # Combines GIL + CPU checks PROFILING_MODE_EXCEPTION = 4 # Only samples when thread has an active exception +PROFILING_MODE_NAMES = { + PROFILING_MODE_WALL: "wall", + PROFILING_MODE_CPU: "cpu", + PROFILING_MODE_GIL: "gil", + PROFILING_MODE_ALL: "all", + PROFILING_MODE_EXCEPTION: "exception", +} + # Sort mode constants SORT_MODE_NSAMPLES = 0 SORT_MODE_TOTTIME = 1 diff --git a/Lib/profiling/sampling/gecko_collector.py b/Lib/profiling/sampling/gecko_collector.py index 8986194268b3ce4..361f6037f216fdc 100644 --- a/Lib/profiling/sampling/gecko_collector.py +++ b/Lib/profiling/sampling/gecko_collector.py @@ -1,8 +1,10 @@ import itertools +import io import json import os import platform import sys +import tempfile import threading import time @@ -61,8 +63,81 @@ PROCESS_TYPE_MAIN = 0 STACKWALK_DISABLED = 0 +# In-memory buffer before spilling to disk +DEFAULT_SPILL_BUFFER_BYTES = 128 * 1024 +_JSON_SEPARATORS = (",", ":") +_JSON_ENCODER = json.JSONEncoder( + separators=_JSON_SEPARATORS, allow_nan=False +) + + +class SpillColumn: + def __init__(self, directory, basename, *, + buffer_bytes=None): + self.path = os.path.join(directory, basename) + self.buffer = bytearray() + self._buffer_bytes = ( + DEFAULT_SPILL_BUFFER_BYTES if buffer_bytes is None + else buffer_bytes + ) + + def append(self, value): + self.buffer += (_JSON_ENCODER.encode(value) + "\n").encode("utf-8") + if len(self.buffer) >= self._buffer_bytes: + self.flush() + + def flush(self): + with open(self.path, "ab") as file: + file.write(self.buffer) + self.buffer.clear() + + def iter_tokens(self): + with open(self.path, encoding="utf-8") as file: + for line in file: + yield line.rstrip("\n") + + +class GeckoThreadSpill: + _COLUMNS = ( + ("samples_stack", "samples-stack.json"), + ("samples_time", "samples-time.json"), + ("markers_name", "markers-name.json"), + ("markers_start_time", "markers-start-time.json"), + ("markers_end_time", "markers-end-time.json"), + ("markers_phase", "markers-phase.json"), + ("markers_category", "markers-category.json"), + ("markers_data", "markers-data.json"), + ) + + def __init__(self, directory, tid): + prefix = f"thread-{tid}-" + for attr, basename in self._COLUMNS: + setattr(self, attr, SpillColumn(directory, prefix + basename)) + self.sample_count = 0 + self.marker_count = 0 + + def append_sample(self, stack_index, time_ms): + self.samples_stack.append(stack_index) + self.samples_time.append(time_ms) + self.sample_count += 1 + + def append_marker(self, name_idx, start_time, end_time, phase, category, data): + self.markers_name.append(name_idx) + self.markers_start_time.append(start_time) + self.markers_end_time.append(end_time) + self.markers_phase.append(phase) + self.markers_category.append(category) + self.markers_data.append(data) + self.marker_count += 1 + + def prepare_read(self): + for attr, _basename in self._COLUMNS: + getattr(self, attr).flush() + class GeckoCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle @@ -75,6 +150,8 @@ def __init__(self, sample_interval_usec, *, skip_idle=False, opcodes=False): # Per-thread data structures self.threads = {} # tid -> thread data + self.spill_dir = None + self.exported = False # Global tables self.libs = [] @@ -149,6 +226,9 @@ def collect(self, stack_frames, timestamps_us=None): stack_frames: List of interpreter/thread frame info timestamps_us: List of timestamps in microseconds (None for live sampling) """ + if self.exported: + raise RuntimeError("cannot append to GeckoCollector after export") + # Handle live sampling (no timestamps provided) if timestamps_us is None: current_time = (time.monotonic() * 1000) - self.start_time @@ -257,15 +337,9 @@ def collect(self, stack_frames, timestamps_us=None): stack_index = self._process_stack(thread_data, frames) # Add samples with timestamps - samples = thread_data["samples"] - samples_stack = samples["stack"] - samples_time = samples["time"] - samples_delay = samples["eventDelay"] - + thread_spill = thread_data["_spill"] for t in times: - samples_stack.append(stack_index) - samples_time.append(t) - samples_delay.append(None) + thread_spill.append_sample(stack_index, t) # Handle opcodes if self.opcodes_enabled and frames: @@ -292,6 +366,8 @@ def collect(self, stack_frames, timestamps_us=None): def _create_thread(self, tid, is_main_thread): """Create a new thread structure with processed profile format.""" + if self.spill_dir is None: + self.spill_dir = tempfile.TemporaryDirectory() thread = { "name": f"Thread-{tid}", @@ -305,15 +381,6 @@ def _create_thread(self, tid, is_main_thread): "tid": tid, "processType": "default", "processName": "Python Process", - # Sample data - processed format with direct arrays - "samples": { - "stack": [], - "time": [], - "eventDelay": [], - "weight": None, - "weightType": "samples", - "length": 0, # Will be updated on export - }, # Stack table - processed format "stackTable": { "frame": [], @@ -364,21 +431,12 @@ def _create_thread(self, tid, is_main_thread): "functionSize": [], "length": 0, }, - # Markers - processed format (arrays) - "markers": { - "data": [], - "name": [], - "startTime": [], - "endTime": [], - "phase": [], - "category": [], - "length": 0, - }, # Caches for deduplication "_stackCache": {}, "_frameCache": {}, "_funcCache": {}, "_resourceCache": {}, + "_spill": GeckoThreadSpill(self.spill_dir.name, tid), } return thread @@ -403,51 +461,42 @@ def _add_marker(self, tid, name, start_time, end_time, category): if tid not in self.threads: return - thread_data = self.threads[tid] duration = end_time - start_time name_idx = self._intern_string(name) - markers = thread_data["markers"] - markers["name"].append(name_idx) - markers["startTime"].append(start_time) - markers["endTime"].append(end_time) - markers["phase"].append(1) # 1 = interval marker - markers["category"].append(category) - markers["data"].append({ - "type": name.replace(" ", ""), - "duration": duration, - "tid": tid - }) - - def _add_opcode_interval_marker(self, tid, opcode, lineno, col_offset, funcname, start_time, end_time): + self.threads[tid]["_spill"].append_marker( + name_idx, start_time, end_time, 1, category, { + "type": name.replace(" ", ""), + "duration": duration, + "tid": tid, + } + ) + + def _add_opcode_interval_marker(self, tid, opcode, lineno, col_offset, + funcname, start_time, end_time): """Add an interval marker for opcode execution span.""" if tid not in self.threads or opcode is None: return - thread_data = self.threads[tid] opcode_info = get_opcode_info(opcode) # Use formatted opcode name (with base opcode for specialized ones) formatted_opname = format_opcode(opcode) name_idx = self._intern_string(formatted_opname) - markers = thread_data["markers"] - markers["name"].append(name_idx) - markers["startTime"].append(start_time) - markers["endTime"].append(end_time) - markers["phase"].append(1) # 1 = interval marker - markers["category"].append(CATEGORY_OPCODES) - markers["data"].append({ - "type": "Opcode", - "opcode": opcode, - "opname": formatted_opname, - "base_opname": opcode_info["base_opname"], - "is_specialized": opcode_info["is_specialized"], - "line": lineno, - "column": col_offset if col_offset >= 0 else None, - "function": funcname, - "duration": end_time - start_time, - }) + self.threads[tid]["_spill"].append_marker( + name_idx, start_time, end_time, 1, CATEGORY_OPCODES, { + "type": "Opcode", + "opcode": opcode, + "opname": formatted_opname, + "base_opname": opcode_info["base_opname"], + "is_specialized": opcode_info["is_specialized"], + "line": lineno, + "column": col_offset if col_offset >= 0 else None, + "function": funcname, + "duration": end_time - start_time, + } + ) def _process_stack(self, thread_data, frames): """Process a stack and return the stack index.""" @@ -658,7 +707,6 @@ def _finalize_markers(self): def export(self, filename): """Export the profile to a Gecko JSON file.""" - if self.sample_count > 0 and self.last_sample_time > 0: self.interval = self.last_sample_time / self.sample_count @@ -679,19 +727,30 @@ def spin(): spinner_thread = threading.Thread(target=spin, daemon=True) spinner_thread.start() + temp_path = None + replaced = False try: - # Finalize any open markers before building profile - self._finalize_markers() - - profile = self._build_profile() - - with open(filename, "w") as f: - json.dump(profile, f, separators=(",", ":")) + self._prepare_for_serialization() + output_dir = os.path.dirname(os.path.abspath(filename)) or "." + with tempfile.NamedTemporaryFile( + "w", dir=output_dir, delete=False + ) as file: + temp_path = file.name + self._stream_profile(file) + os.replace(temp_path, filename) + replaced = True finally: + self.exported = True stop_spinner.set() spinner_thread.join(timeout=1.0) # Small delay to ensure the clear happens time.sleep(0.01) + if temp_path is not None and not replaced: + try: + os.unlink(temp_path) + except FileNotFoundError: + pass + self._cleanup_spills() print(f"Gecko profile written to {filename}") print( @@ -725,34 +784,17 @@ def _build_marker_schema(self): def _build_profile(self): """Build the complete profile structure in processed format.""" - # Convert thread data to final format - threads = [] - - for tid, thread_data in self.threads.items(): - # Update lengths - samples = thread_data["samples"] - stack_table = thread_data["stackTable"] - frame_table = thread_data["frameTable"] - func_table = thread_data["funcTable"] - resource_table = thread_data["resourceTable"] - - samples["length"] = len(samples["stack"]) - stack_table["length"] = len(stack_table["frame"]) - frame_table["length"] = len(frame_table["func"]) - func_table["length"] = len(func_table["name"]) - resource_table["length"] = len(resource_table["name"]) - thread_data["markers"]["length"] = len(thread_data["markers"]["name"]) - - # Clean up internal caches - del thread_data["_stackCache"] - del thread_data["_frameCache"] - del thread_data["_funcCache"] - del thread_data["_resourceCache"] - - threads.append(thread_data) - - # Main profile structure in processed format - profile = { + try: + self._prepare_for_serialization() + file = io.StringIO() + self._stream_profile(file) + return json.loads(file.getvalue()) + finally: + self.exported = True + self._cleanup_spills() + + def _profile_head(self): + return { "meta": { "interval": self.interval, "startTime": self.start_time, @@ -782,7 +824,10 @@ def _build_profile(self): }, }, "libs": self.libs, - "threads": threads, + } + + def _profile_tail(self): + return { "pages": [], "shared": { "stringArray": self.global_strings, @@ -790,4 +835,146 @@ def _build_profile(self): }, } - return profile + def _prepare_for_serialization(self): + if self.exported: + raise RuntimeError("GeckoCollector has already been exported") + self._finalize_markers() + for thread_data in self.threads.values(): + thread_data["_spill"].prepare_read() + thread_data["stackTable"]["length"] = len(thread_data["stackTable"]["frame"]) + thread_data["frameTable"]["length"] = len(thread_data["frameTable"]["func"]) + thread_data["funcTable"]["length"] = len(thread_data["funcTable"]["name"]) + thread_data["resourceTable"]["length"] = len(thread_data["resourceTable"]["name"]) + + def _cleanup_spills(self): + if self.spill_dir is not None: + self.spill_dir.cleanup() + self.spill_dir = None + + def _stream_profile(self, file): + file.write("{") + first = True + for key, value in self._profile_head().items(): + first = _write_json_member(file, key, value, first) + + first = _write_member_name(file, "threads", first) + file.write("[") + for index, (tid, thread_data) in enumerate(self.threads.items()): + if index: + file.write(",") + self._stream_thread(file, tid, thread_data) + file.write("]") + + for key, value in self._profile_tail().items(): + first = _write_json_member(file, key, value, first) + file.write("}") + + def _stream_thread(self, file, tid, thread_data): + spill = thread_data["_spill"] + metadata = { + "name": thread_data["name"], + "isMainThread": thread_data["isMainThread"], + "processStartupTime": thread_data["processStartupTime"], + "processShutdownTime": thread_data["processShutdownTime"], + "registerTime": thread_data["registerTime"], + "unregisterTime": thread_data["unregisterTime"], + "pausedRanges": thread_data["pausedRanges"], + "pid": thread_data["pid"], + "tid": thread_data["tid"], + "processType": thread_data["processType"], + "processName": thread_data["processName"], + } + file.write("{") + first = True + for key, value in metadata.items(): + first = _write_json_member(file, key, value, first) + + first = _write_member_name(file, "samples", first) + self._stream_samples(file, spill) + for key in ( + "stackTable", + "frameTable", + "funcTable", + "resourceTable", + "nativeSymbols", + ): + first = _write_json_member(file, key, thread_data[key], first) + first = _write_member_name(file, "markers", first) + self._stream_markers(file, spill) + file.write("}") + + def _stream_samples(self, file, spill): + _stream_column_table( + file, + ( + ("stack", spill.samples_stack.iter_tokens()), + ("time", spill.samples_time.iter_tokens()), + ("eventDelay", ("null" for _ in range(spill.sample_count))), + ), + spill.sample_count, + ( + ("weight", None), + ("weightType", "samples"), + ("length", spill.sample_count), + ), + ) + + def _stream_markers(self, file, spill): + _stream_column_table( + file, + ( + ("data", spill.markers_data.iter_tokens()), + ("name", spill.markers_name.iter_tokens()), + ("startTime", spill.markers_start_time.iter_tokens()), + ("endTime", spill.markers_end_time.iter_tokens()), + ("phase", spill.markers_phase.iter_tokens()), + ("category", spill.markers_category.iter_tokens()), + ), + spill.marker_count, + (("length", spill.marker_count),), + ) + + +def _write_json(file, value): + for chunk in _JSON_ENCODER.iterencode(value): + file.write(chunk) + + +def _write_member_name(file, name, first): + if not first: + file.write(",") + _write_json(file, name) + file.write(":") + return False + + +def _write_json_member(file, name, value, first): + first = _write_member_name(file, name, first) + _write_json(file, value) + return first + + +def _stream_column_table(file, columns, expected_count, trailing_members=()): + file.write("{") + first = True + for name, token_iter in columns: + first = _write_member_name(file, name, first) + _stream_array(file, token_iter, expected_count, name) + for name, value in trailing_members: + first = _write_json_member(file, name, value, first) + file.write("}") + + +def _stream_array(file, token_iter, expected_count, label="array"): + file.write("[") + count = 0 + for token in token_iter: + if count: + file.write(",") + file.write(token) + count += 1 + if count != expected_count: + raise RuntimeError( + f"streamed {count} {label} items, expected {expected_count}" + ) + file.write("]") diff --git a/Lib/profiling/sampling/heatmap_collector.py b/Lib/profiling/sampling/heatmap_collector.py index 5c36d78f5535e71..78f1e39f6a002d0 100644 --- a/Lib/profiling/sampling/heatmap_collector.py +++ b/Lib/profiling/sampling/heatmap_collector.py @@ -5,11 +5,9 @@ import html import importlib.resources import json -import locale import math import os import platform -import site import sys from dataclasses import dataclass, field from pathlib import Path @@ -452,7 +450,8 @@ def process_frames(self, frames, thread_id, weight=1): next_lineno = extract_lineno(next_frame[1]) self._record_call_relationship( (filename, lineno, funcname), - (next_frame[0], next_lineno, next_frame[2]) + (next_frame[0], next_lineno, next_frame[2]), + weight=weight, ) def _is_valid_frame(self, filename, lineno): @@ -561,7 +560,7 @@ def _get_bytecode_data_for_line(self, filename, lineno): result.sort(key=lambda x: (-x['samples'], x['opcode'])) return result - def _record_call_relationship(self, callee_frame, caller_frame): + def _record_call_relationship(self, callee_frame, caller_frame, weight=1): """Record caller/callee relationship between adjacent frames.""" callee_filename, callee_lineno, callee_funcname = callee_frame caller_filename, caller_lineno, caller_funcname = caller_frame @@ -587,7 +586,7 @@ def _record_call_relationship(self, callee_frame, caller_frame): # Count this call edge for path analysis edge_key = (caller_key, callee_key) - self.edge_samples[edge_key] += 1 + self.edge_samples[edge_key] += weight def export(self, output_path): """Export heatmap data as HTML files in a directory. diff --git a/Lib/profiling/sampling/jsonl_collector.py b/Lib/profiling/sampling/jsonl_collector.py new file mode 100644 index 000000000000000..5aa42ef09024dc3 --- /dev/null +++ b/Lib/profiling/sampling/jsonl_collector.py @@ -0,0 +1,267 @@ +"""JSON Lines (JSONL) collector for the sampling profiler. + +Emits a normalized newline-delimited JSON record stream suitable for +programmatic consumption by external tools, scripts, and agents. Each line +is one JSON object; consumers can parse the file incrementally line by +line, but the producer writes the whole file at the end of the run (it is +not a live/streaming producer). + +Record schema +============= + +Every record is a JSON object with at least ``"type"``, ``"v"`` (record +schema version), and ``"run_id"`` (UUID4 hex tagging the run; allows +demultiplexing concatenated streams). Records appear in this fixed order: + +1. ``meta`` (exactly one, first line):: + + {"type":"meta","v":0,"run_id":"", + "sample_interval_usec":,"mode":"wall|cpu|gil|all|exception"} + + ``mode`` is omitted when not provided. + +2. ``string_table`` (zero or more):: + + {"type":"string_table","v":0,"run_id":"", + "strings":[{"str_id":,"value":""}, ...]} + + Strings (filenames, function names) are interned to keep repeated values + compact. IDs are zero-based. Each chunk holds up to ``_CHUNK_SIZE`` + entries, and each entry carries its explicit ``str_id`` so consumers do + not need to infer offsets across chunks. + +3. ``frame_table`` (zero or more):: + + {"type":"frame_table","v":0,"run_id":"", + "frames":[{"frame_id":,"path_str_id":,"func_str_id":, + "line":,"end_line":,"col":, + "end_col":}, ...]} + + ``end_line``/``col``/``end_col`` are *omitted* when source location data + is unavailable (a missing key means "not available", not zero or null). + ``line`` is ``0`` for synthetic frames (for example, internal marker + frames whose source location is None). Frame IDs are zero-based. + +4. ``agg`` (zero or more):: + + {"type":"agg","v":0,"run_id":"","kind":"frame","scope":"final", + "samples_total":, + "entries":[{"frame_id":,"self":,"cumulative":}, ...]} + + ``self`` counts samples where the frame was the leaf (currently + executing); ``cumulative`` counts samples where the frame appeared + anywhere in the stack (deduped per sample so recursion does not + double-count). ``samples_total`` is the run-wide total, repeated on + each chunk so a streaming consumer always knows the denominator. + +5. ``end`` (exactly one, last line):: + + {"type":"end","v":0,"run_id":"","samples_total":} + + Presence of ``end`` is the consumer's signal that the file is complete. + +Forward compatibility +===================== + +Consumers MUST ignore unknown record ``"type"`` values and unknown object +fields. New fields will be added by adding optional keys; an incompatible +schema change will bump the per-record ``"v"``. +""" + +from collections import Counter +import json +import uuid +from itertools import batched + +from .constants import PROFILING_MODE_NAMES +from .collector import normalize_location +from .stack_collector import StackTraceCollector + + +_CHUNK_SIZE = 256 +_SCHEMA_VERSION = 0 + + +class JsonlCollector(StackTraceCollector): + """Collector that exports finalized profiling data as JSONL. + + See the module docstring for the full record schema. The collector + accumulates samples in memory and writes the complete file at + ``export()`` time. + """ + + def __init__(self, sample_interval_usec, *, skip_idle=False, mode=None): + super().__init__(sample_interval_usec, skip_idle=skip_idle) + self.run_id = uuid.uuid4().hex + + self._string_to_id = {} + self._strings = [] + + self._frame_to_id = {} + self._frames = [] + + self._frame_self = Counter() + self._frame_cumulative = Counter() + self._samples_total = 0 + self._seen_frame_ids = set() + + self._mode = mode + + def process_frames(self, frames, _thread_id, weight=1): + self._samples_total += weight + self._seen_frame_ids.clear() + + for i, (filename, location, funcname, _opcode) in enumerate(frames): + frame_id = self._get_or_create_frame_id( + filename, location, funcname + ) + is_leaf = i == 0 + count_cumulative = frame_id not in self._seen_frame_ids + + if count_cumulative: + self._seen_frame_ids.add(frame_id) + + if is_leaf: + self._frame_self[frame_id] += weight + + if count_cumulative: + self._frame_cumulative[frame_id] += weight + + def export(self, filename): + with open(filename, "w", encoding="utf-8") as output: + self._write_message(output, self._build_meta_record()) + self._write_chunked_records( + output, + { + "type": "string_table", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + }, + "strings", + self._strings, + ) + self._write_chunked_records( + output, + { + "type": "frame_table", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + }, + "frames", + self._frames, + ) + self._write_chunked_records( + output, + { + "type": "agg", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + "kind": "frame", + "scope": "final", + "samples_total": self._samples_total, + }, + "entries", + self._iter_final_agg_entries(), + ) + self._write_message(output, self._build_end_record()) + print(f"JSONL profile written to {filename}") + + def _build_meta_record(self): + record = { + "type": "meta", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + "sample_interval_usec": self.sample_interval_usec, + } + + if self._mode is not None: + record["mode"] = PROFILING_MODE_NAMES.get( + self._mode, str(self._mode) + ) + + return record + + def _build_end_record(self): + record = { + "type": "end", + "v": _SCHEMA_VERSION, + "run_id": self.run_id, + "samples_total": self._samples_total, + } + + return record + + def _iter_final_agg_entries(self): + for frame_record in self._frames: + frame_id = frame_record["frame_id"] + yield { + "frame_id": frame_id, + "self": self._frame_self[frame_id], + "cumulative": self._frame_cumulative[frame_id], + } + + def _get_or_create_frame_id(self, filename, location, funcname): + location_fields = self._location_to_export_fields(location) + func_str_id = self._intern_string(funcname) + path_str_id = self._intern_string(filename) + + frame_key = ( + path_str_id, + func_str_id, + location_fields["line"], + location_fields.get("end_line"), + location_fields.get("col"), + location_fields.get("end_col"), + ) + + if (frame_id := self._frame_to_id.get(frame_key)) is not None: + return frame_id + + frame_id = len(self._frames) + frame_record = { + "frame_id": frame_id, + "path_str_id": path_str_id, + "func_str_id": func_str_id, + **location_fields, + } + + self._frame_to_id[frame_key] = frame_id + self._frames.append(frame_record) + return frame_id + + def _intern_string(self, value): + value = str(value) + + if (string_id := self._string_to_id.get(value)) is not None: + return string_id + + string_id = len(self._strings) + self._string_to_id[value] = string_id + self._strings.append({"str_id": string_id, "value": value}) + return string_id + + @staticmethod + def _location_to_export_fields(location): + lineno, end_lineno, col_offset, end_col_offset = normalize_location( + location + ) + + fields = {"line": lineno} + if end_lineno > 0: + fields["end_line"] = end_lineno + if col_offset >= 0: + fields["col"] = col_offset + if end_col_offset >= 0: + fields["end_col"] = end_col_offset + return fields + + def _write_chunked_records( + self, output, base_record, chunk_field, entries + ): + for chunk in batched(entries, _CHUNK_SIZE): + self._write_message(output, {**base_record, chunk_field: chunk}) + + @staticmethod + def _write_message(output, record): + output.write(json.dumps(record, separators=(",", ":"))) + output.write("\n") diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index c03df4075277cdb..2805134f239eb38 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -9,7 +9,7 @@ import sys import sysconfig import time -import _colorize +lazy import _colorize from ..collector import Collector, extract_lineno from ..constants import ( @@ -18,9 +18,6 @@ THREAD_STATUS_UNKNOWN, THREAD_STATUS_GIL_REQUESTED, THREAD_STATUS_HAS_EXCEPTION, - PROFILING_MODE_CPU, - PROFILING_MODE_GIL, - PROFILING_MODE_WALL, ) from .constants import ( MICROSECONDS_PER_SECOND, diff --git a/Lib/profiling/sampling/live_collector/display.py b/Lib/profiling/sampling/live_collector/display.py index f5324421b102111..ec7e39c02963056 100644 --- a/Lib/profiling/sampling/live_collector/display.py +++ b/Lib/profiling/sampling/live_collector/display.py @@ -1,6 +1,5 @@ """Display interface abstractions for the live profiling collector.""" -import contextlib import curses from abc import ABC, abstractmethod diff --git a/Lib/profiling/sampling/live_collector/widgets.py b/Lib/profiling/sampling/live_collector/widgets.py index 86d2649f875e622..7535d0c753b0465 100644 --- a/Lib/profiling/sampling/live_collector/widgets.py +++ b/Lib/profiling/sampling/live_collector/widgets.py @@ -1,7 +1,6 @@ """Widget classes for the live profiling collector UI.""" import curses -import time from abc import ABC, abstractmethod from .constants import ( @@ -13,24 +12,14 @@ WIDTH_THRESHOLD_CUMUL_PCT, WIDTH_THRESHOLD_CUMTIME, MICROSECONDS_PER_SECOND, - DISPLAY_UPDATE_INTERVAL_SEC, MIN_BAR_WIDTH, MAX_SAMPLE_RATE_BAR_WIDTH, MAX_EFFICIENCY_BAR_WIDTH, MIN_SAMPLE_RATE_FOR_SCALING, FOOTER_LINES, - FINISHED_BANNER_EXTRA_LINES, OPCODE_PANEL_HEIGHT, ) -from ..constants import ( - THREAD_STATUS_HAS_GIL, - THREAD_STATUS_ON_CPU, - THREAD_STATUS_UNKNOWN, - THREAD_STATUS_GIL_REQUESTED, - PROFILING_MODE_CPU, - PROFILING_MODE_GIL, - PROFILING_MODE_WALL, -) +from ..constants import PROFILING_MODE_GIL from ..opcode_utils import get_opcode_info, format_opcode diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 6be1d698ffaa9a5..43b1daf2a119d4e 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -1,13 +1,15 @@ import collections import marshal import pstats +lazy from _colorize import ANSIColors -from _colorize import ANSIColors from .collector import Collector, extract_lineno from .constants import MICROSECONDS_PER_SECOND, PROFILING_MODE_CPU class PstatsCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.result = collections.defaultdict( lambda: dict(total_rec_calls=0, direct_calls=0, cumulative_calls=0) diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 9e315c080c353db..6cb6dd483520887 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -6,12 +6,8 @@ import sysconfig import time from collections import deque -from _colorize import ANSIColors +lazy from _colorize import ANSIColors -from .pstats_collector import PstatsCollector -from .stack_collector import CollapsedStackCollector, FlamegraphCollector -from .heatmap_collector import HeatmapCollector -from .gecko_collector import GeckoCollector from .binary_collector import BinaryCollector @@ -47,9 +43,41 @@ def _pause_threads(unwinder, blocking): # If fewer samples are collected, we skip the TUI and just print a message MIN_SAMPLES_FOR_TUI = 200 +# Maximum number of consecutive identical samples to keep before flushing. +MAX_PENDING_SAMPLES = 8192 + + +def _resolve_python_pid(pid): + """On Windows, if pid is a venvlauncher process, return the child Python PID. + + The venvlauncher (used as python.exe in venvs) spawns the real Python + interpreter as a child process via CreateProcessW. The RemoteUnwinder + needs the child's PID, not the launcher's. + + Returns the original pid if not on Windows, not a venv launcher, + or no child process is found. + """ + if os.name != "nt" or sys.prefix == sys.base_prefix: + return pid + try: + children = _remote_debugging.get_child_pids(pid, recursive=False) + python_children = [ + child for child in children + if _remote_debugging.is_python_process(child) + ] + if len(python_children) == 1: + return python_children[0] + except (OSError, RuntimeError) as err: + raise SystemExit( + f"Failed to initialize profiler from virtualenv: {err}\n" + f"Try running with the base interpreter: {sys._base_executable}" + ) from err + return pid + + class SampleProfiler: def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MODE_WALL, native=False, gc=True, opcodes=False, skip_non_matching_threads=True, collect_stats=False, blocking=False): - self.pid = pid + self.pid = _resolve_python_pid(pid) self.sample_interval_usec = sample_interval_usec self.all_threads = all_threads self.mode = mode # Store mode for later use @@ -58,10 +86,6 @@ def __init__(self, pid, sample_interval_usec, all_threads, *, mode=PROFILING_MOD try: self.unwinder = self._new_unwinder(native, gc, opcodes, skip_non_matching_threads) except RuntimeError as err: - if os.name == "nt" and sys.executable.endswith("python.exe"): - raise SystemExit( - "Running profiling.sampling from virtualenv on Windows platform is not supported" - ) from err raise SystemExit(err) from err # Track sample intervals and total sample count self.sample_intervals = deque(maxlen=100) @@ -109,6 +133,20 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): last_sample_time = start_time realtime_update_interval = 1.0 # Update every second last_realtime_update = start_time + aggregating = getattr(collector, 'aggregating', False) is True + prev_stack = None + pending_count = 0 + pending_timestamps = [] if aggregating else None + + def flush_pending(): + nonlocal pending_count, pending_timestamps + if pending_count == 0: + return + pending_count = 0 + ts = pending_timestamps + pending_timestamps = [] + collector.collect(prev_stack, timestamps_us=ts) + try: while duration_sec is None or running_time_sec < duration_sec: # Check if live collector wants to stop @@ -116,6 +154,7 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): break current_time = time.perf_counter() + current_time_us = int(current_time * 1_000_000) if next_time > current_time: sleep_time = (next_time - current_time) * 0.9 if sleep_time > 0.0001: @@ -125,13 +164,24 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): stack_frames = self._get_stack_trace( async_aware=async_aware ) - collector.collect(stack_frames) + if aggregating: + if stack_frames != prev_stack: + flush_pending() + prev_stack = stack_frames + pending_count += 1 + pending_timestamps.append(current_time_us) + if pending_count >= MAX_PENDING_SAMPLES: + flush_pending() + else: + collector.collect(stack_frames) except ProcessLookupError as e: running_time_sec = current_time - start_time break except (RuntimeError, UnicodeDecodeError, MemoryError, OSError): + flush_pending() collector.collect_failed_sample() errors += 1 + prev_stack = None except Exception as e: if not _is_process_running(self.pid): break @@ -163,6 +213,8 @@ def sample(self, collector, duration_sec=None, *, async_aware=False): interrupted = True running_time_sec = time.perf_counter() - start_time print("Interrupted by user.") + finally: + flush_pending() # Clear real-time stats line if it was being displayed if self.realtime_stats and len(self.sample_intervals) > 0: @@ -296,6 +348,33 @@ def _print_unwinder_stats(self): print(f" Hits: {code_hits:n} ({ANSIColors.GREEN}{fmt(code_hits_pct)}%{ANSIColors.RESET})") print(f" Misses: {code_misses:n} ({ANSIColors.RED}{fmt(code_misses_pct)}%{ANSIColors.RESET})") + batched_attempts = stats.get('batched_read_attempts', 0) + batched_successes = stats.get('batched_read_successes', 0) + batched_misses = stats.get('batched_read_misses', 0) + segments_requested = stats.get('batched_read_segments_requested', 0) + segments_completed = stats.get('batched_read_segments_completed', 0) + if batched_attempts > 0: + batched_success_rate = stats.get('batched_read_success_rate', 0.0) + batched_miss_rate = 100.0 - batched_success_rate + segment_completion_rate = stats.get( + 'batched_read_segment_completion_rate', 0.0 + ) + + print(f" {ANSIColors.CYAN}Batched Reads:{ANSIColors.RESET}") + print(f" Attempts: {batched_attempts:n}") + print( + f" Successes: {batched_successes:n} " + f"({ANSIColors.GREEN}{fmt(batched_success_rate)}%{ANSIColors.RESET})" + ) + print( + f" Misses: {batched_misses:n} " + f"({ANSIColors.RED}{fmt(batched_miss_rate)}%{ANSIColors.RESET})" + ) + print( + f" Segments read: {segments_completed:n}/{segments_requested:n} " + f"({ANSIColors.GREEN}{fmt(segment_completion_rate)}%{ANSIColors.RESET})" + ) + # Memory operations memory_reads = stats.get('memory_reads', 0) memory_bytes = stats.get('memory_bytes_read', 0) diff --git a/Lib/profiling/sampling/stack_collector.py b/Lib/profiling/sampling/stack_collector.py index 04622a8c1e89ef6..42281dc6454c83c 100644 --- a/Lib/profiling/sampling/stack_collector.py +++ b/Lib/profiling/sampling/stack_collector.py @@ -16,6 +16,8 @@ class StackTraceCollector(Collector): + aggregating = True + def __init__(self, sample_interval_usec, *, skip_idle=False): self.sample_interval_usec = sample_interval_usec self.skip_idle = skip_idle @@ -698,6 +700,8 @@ def _add_elided_metadata(self, node, baseline_stats, scale, path): func_key = self._extract_func_key(node, self._baseline_collector._string_table) current_path = path + (func_key,) if func_key else path + baseline_self = 0 + baseline_total = 0 if func_key and current_path in baseline_stats: baseline_data = baseline_stats[current_path] baseline_self = baseline_data["self"] * scale diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 694ea9304da9f94..b22446e2f098dee 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -177,7 +177,7 @@ def main(): import argparse description = 'A simple command-line interface for py_compile module.' - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) parser.add_argument( '-q', '--quiet', action='store_true', @@ -202,12 +202,12 @@ def main(): if args.quiet: parser.exit(1) else: - parser.exit(1, error.msg) + parser.exit(1, error.msg + '\n') except OSError as error: if args.quiet: parser.exit(1) else: - parser.exit(1, str(error)) + parser.exit(1, str(error) + '\n') if __name__ == "__main__": diff --git a/Lib/pydoc.py b/Lib/pydoc.py index a1a6aad434ddf4d..041ee26c791f36b 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -1240,6 +1240,17 @@ def indent(self, text, prefix=' '): lines = [(prefix + line).rstrip() for line in text.split('\n')] return '\n'.join(lines) + def _format_doc(self, text, width=68): + """Wraps the single-line summary if it is too long.""" + if not text: return '' + lines = text.split('\n', 2) + if len(lines) > 1 and lines[1]: + return text + lines[:1] = textwrap.wrap(lines[0], width, + break_long_words=False, + break_on_hyphens=False) + return '\n'.join(lines) + def section(self, title, contents): """Format a section with a given heading.""" clean_contents = self.indent(contents).rstrip() @@ -1344,7 +1355,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): if data: contents = [] for key, value in data: - contents.append(self.docother(value, key, name, maxlen=70)) + contents.append(self.docother(value, key, name, maxlen=76)) result = result + self.section('DATA', '\n'.join(contents)) if version := self._get_version(object): @@ -1390,6 +1401,7 @@ def makename(c, m=object.__module__): doc = getdoc(object) if doc: + doc = self._format_doc(doc) push(doc + '\n') # List the mro, if non-trivial. @@ -1466,7 +1478,7 @@ def spilldata(msg, attrs, predicate): obj = getattr(object, name) except AttributeError: obj = homecls.__dict__[name] - push(self.docother(obj, name, mod, maxlen=70, doc=doc) + + push(self.docother(obj, name, mod, maxlen=72, doc=doc) + '\n') return attrs @@ -1590,6 +1602,7 @@ def docroutine(self, object, name=None, mod=None, cl=None, homecls=None): return decl + '\n' else: doc = getdoc(object) or '' + doc = self._format_doc(doc) return decl + '\n' + (doc and self.indent(doc).rstrip() + '\n') def docdata(self, object, name=None, mod=None, cl=None, *ignored): @@ -1602,6 +1615,7 @@ def docdata(self, object, name=None, mod=None, cl=None, *ignored): push('\n') doc = getdoc(object) or '' if doc: + doc = self._format_doc(doc) push(self.indent(doc)) push('\n') return ''.join(results) @@ -1615,12 +1629,13 @@ def docother(self, object, name=None, mod=None, parent=None, *ignored, if maxlen: line = (name and name + ' = ' or '') + repr chop = maxlen - len(line) - if chop < 0: repr = repr[:chop] + '...' + if chop < 0: repr = repr[:chop-3] + '...' line = (name and self.bold(name) + ' = ' or '') + repr if not doc: doc = getdoc(object) if doc: - line += '\n' + self.indent(str(doc)) + '\n' + doc = self._format_doc(str(doc)) + line += '\n' + self.indent(doc) + '\n' return line class _PlainTextDoc(TextDoc): @@ -1829,6 +1844,7 @@ class Helper: 'async': ('async', ''), 'await': ('await', ''), 'break': ('break', 'while for'), + 'case': 'match', 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), 'def': ('function', ''), @@ -1840,11 +1856,13 @@ class Helper: 'for': ('for', 'break continue while'), 'from': 'import', 'global': ('global', 'nonlocal NAMESPACES'), - 'if': ('if', 'TRUTHVALUE'), + 'if': ('if', 'TRUTHVALUE match'), 'import': ('import', 'MODULES'), 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), + 'lazy': ('lazy', 'MODULES'), + 'match': ('match', 'if'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', 'or': 'BOOLEAN', diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py index 9e33c6765bb19ee..1a3126d3db95909 100644 --- a/Lib/pydoc_data/module_docs.py +++ b/Lib/pydoc_data/module_docs.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Apr 7 14:21:08 2026 +# Autogenerated by Sphinx on Thu May 7 16:26:23 2026 # as part of the release process. module_docs = { diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 68f6771d4cedcfe..5f61001c46b79ce 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Apr 7 14:21:08 2026 +# Autogenerated by Sphinx on Thu May 7 16:26:23 2026 # as part of the release process. topics = { @@ -2572,7 +2572,7 @@ def foo(): | "None" | "True" | "False" - signed_number: ["-"] NUMBER + signed_number: ["+" | "-"] NUMBER The rule "strings" and the token "NUMBER" are defined in the standard Python grammar. Triple-quoted strings are supported. Raw strings and @@ -6841,8 +6841,9 @@ def whats_on_the_telly(penguin=None): 1. find a module, loading and initializing it if necessary -2. define a name or names in the local namespace for the scope where - the "import" statement occurs. +2. define a name or names in the current namespace for the scope where + the "import" statement occurs, just as an assignment statement + would (including "global" and "nonlocal" semantics). When the statement contains multiple clauses (separated by commas) the two steps are carried out separately for each clause, just as though @@ -6887,7 +6888,7 @@ def whats_on_the_telly(penguin=None): 3. if the attribute is not found, "ImportError" is raised. - 4. otherwise, a reference to that value is stored in the local + 4. otherwise, a reference to that value is stored in the current namespace, using the name in the "as" clause if it is present, otherwise using the attribute name @@ -6984,6 +6985,53 @@ def whats_on_the_telly(penguin=None): Added in version 3.15. +Compatibility via "__lazy_modules__" +------------------------------------ + +As an alternative to using the "lazy" keyword, a module can opt into +lazy loading for specific imports by defining a module-level +"__lazy_modules__" variable. When present, it must be a container of +fully qualified module name strings. Any regular (non-"lazy") +"import" statement at module scope whose target appears in +"__lazy_modules__" is treated as a lazy import, exactly as if the +"lazy" keyword had been used. + +This provides a way to enable lazy loading for specific dependencies +without changing individual "import" statements. This is useful when +supporting Python versions older than 3.15 while using lazy imports in +3.15+: + + __lazy_modules__ = ["json", "pathlib"] + + import json # loaded lazily (name is in __lazy_modules__) + import os # loaded eagerly (name not in __lazy_modules__) + + import pathlib # loaded lazily + +Relative imports are resolved to their absolute name before the +lookup, so "__lazy_modules__" must always contain fully qualified +module names. + +For "from"-style imports, the relevant name is the module following +"from", not the names of its members: + + # In mypackage/mymodule.py + __lazy_modules__ = ["mypackage", "mypackage.sub.utils"] + + from . import helper # loaded lazily: . resolves to mypackage + from .sub.utils import func # loaded lazily: .sub.utils resolves to mypackage.sub.utils + import json # loaded eagerly (not in __lazy_modules__) + +Imports inside functions, class bodies, or "try"/"except"/"finally" +blocks are always eager, regardless of "__lazy_modules__". + +Setting "-X lazy_imports=none" (or the "PYTHON_LAZY_IMPORTS" +environment variable to "none") overrides "__lazy_modules__" and +forces all imports to be eager. + +Added in version 3.15. + + Future statements ================= @@ -11035,8 +11083,18 @@ class is used in a class pattern with positional arguments, each str.swapcase() Return a copy of the string with uppercase characters converted to - lowercase and vice versa. Note that it is not necessarily true that - "s.swapcase().swapcase() == s". + lowercase and vice versa. For example: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that "s.swapcase().swapcase() + == s". For example: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also "str.lower()" and "str.upper()". str.title() @@ -12871,6 +12929,19 @@ def foo(): Added in version 3.14. +module.__lazy_modules__ + + A container (an object implementing "__contains__()") of fully + qualified module name strings. When defined at module scope, any + regular "import" statement in that module whose target module name + appears in this container is treated as a lazy import, as if the + "lazy" keyword had been used. Imports inside functions, class + bodies, or "try"/"except"/"finally" blocks are unaffected. + + See Compatibility via __lazy_modules__ for details and examples. + + Added in version 3.15. + Module dictionaries ------------------- @@ -13167,12 +13238,6 @@ class instance has a namespace implemented as a dictionary which is +----------------------------------------------------+----------------------------------------------------+ | codeobject.co_firstlineno | The line number of the first line of the function | +----------------------------------------------------+----------------------------------------------------+ -| codeobject.co_lnotab | A string encoding the mapping from *bytecode* | -| | offsets to line numbers. For details, see the | -| | source code of the interpreter. Deprecated since | -| | version 3.12: This attribute of code objects is | -| | deprecated, and may be removed in Python 3.15. | -+----------------------------------------------------+----------------------------------------------------+ | codeobject.co_stacksize | The required stack size of the code object | +----------------------------------------------------+----------------------------------------------------+ | codeobject.co_flags | An "integer" encoding a number of flags for the | diff --git a/Lib/random.py b/Lib/random.py index 726a71e782893c8..4541267bab866a4 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -1016,26 +1016,26 @@ def _test(N=10_000): def _parse_args(arg_list: list[str] | None): import argparse parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, color=True) + formatter_class=argparse.RawTextHelpFormatter) group = parser.add_mutually_exclusive_group() group.add_argument( "-c", "--choice", nargs="+", help="print a random choice") group.add_argument( "-i", "--integer", type=int, metavar="N", - help="print a random integer between 1 and N inclusive") + help="print a random integer between 1 and `N` inclusive") group.add_argument( "-f", "--float", type=float, metavar="N", - help="print a random floating-point number between 0 and N inclusive") + help="print a random floating-point number between 0 and `N` inclusive") group.add_argument( "--test", type=int, const=10_000, nargs="?", help=argparse.SUPPRESS) parser.add_argument("input", nargs="*", help="""\ if no options given, output depends on the input - string or multiple: same as --choice - integer: same as --integer - float: same as --float""") + string or multiple: same as `--choice` + integer: same as `--integer` + float: same as `--float`""") args = parser.parse_args(arg_list) return args, parser.format_help() diff --git a/Lib/re/_compiler.py b/Lib/re/_compiler.py index c2ca8e25abe34d3..fb0c8d35f6f89a3 100644 --- a/Lib/re/_compiler.py +++ b/Lib/re/_compiler.py @@ -14,13 +14,16 @@ from . import _parser from ._constants import * from ._casefix import _EXTRA_CASES +from ._optimizer import ( + _combine_flags, _compile_charset, _optimize_charset, _compile_info, + _simple, _CHARSET_ALL, _CODEBITS, MAXCODE, +) assert _sre.MAGIC == MAGIC, "SRE module mismatch" _LITERAL_CODES = {LITERAL, NOT_LITERAL} _SUCCESS_CODES = {SUCCESS, FAILURE} _ASSERT_CODES = {ASSERT, ASSERT_NOT} -_UNIT_CODES = _LITERAL_CODES | {ANY, IN} _REPEATING_CODES = { MIN_REPEAT: (REPEAT, MIN_UNTIL, MIN_REPEAT_ONE), @@ -28,14 +31,6 @@ POSSESSIVE_REPEAT: (POSSESSIVE_REPEAT, SUCCESS, POSSESSIVE_REPEAT_ONE), } -_CHARSET_ALL = [(NEGATE, None)] - -def _combine_flags(flags, add_flags, del_flags, - TYPE_FLAGS=_parser.TYPE_FLAGS): - if add_flags & TYPE_FLAGS: - flags &= ~TYPE_FLAGS - return (flags | add_flags) & ~del_flags - def _compile(code, pattern, flags): # internal: compile a (sub)pattern emit = code.append @@ -218,372 +213,6 @@ def _compile(code, pattern, flags): else: raise PatternError(f"internal: unsupported operand type {op!r}") -def _compile_charset(charset, flags, code): - # compile charset subprogram - emit = code.append - for op, av in charset: - emit(op) - if op is NEGATE: - pass - elif op is LITERAL: - emit(av) - elif op is RANGE or op is RANGE_UNI_IGNORE: - emit(av[0]) - emit(av[1]) - elif op is CHARSET: - code.extend(av) - elif op is BIGCHARSET: - code.extend(av) - elif op is CATEGORY: - if flags & SRE_FLAG_LOCALE: - emit(CH_LOCALE[av]) - elif flags & SRE_FLAG_UNICODE: - emit(CH_UNICODE[av]) - else: - emit(av) - else: - raise PatternError(f"internal: unsupported set operator {op!r}") - emit(FAILURE) - -def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): - # internal: optimize character set - out = [] - tail = [] - charmap = bytearray(256) - hascased = False - for op, av in charset: - while True: - try: - if op is LITERAL: - if fixup: # IGNORECASE and not LOCALE - av = fixup(av) - charmap[av] = 1 - if fixes and av in fixes: - for k in fixes[av]: - charmap[k] = 1 - if not hascased and iscased(av): - hascased = True - else: - charmap[av] = 1 - elif op is RANGE: - r = range(av[0], av[1]+1) - if fixup: # IGNORECASE and not LOCALE - if fixes: - for i in map(fixup, r): - charmap[i] = 1 - if i in fixes: - for k in fixes[i]: - charmap[k] = 1 - else: - for i in map(fixup, r): - charmap[i] = 1 - if not hascased: - hascased = any(map(iscased, r)) - else: - for i in r: - charmap[i] = 1 - elif op is NEGATE: - out.append((op, av)) - elif op is CATEGORY and tail and (CATEGORY, CH_NEGATE[av]) in tail: - # Optimize [\s\S] etc. - out = [] if out else _CHARSET_ALL - return out, False - else: - tail.append((op, av)) - except IndexError: - if len(charmap) == 256: - # character set contains non-UCS1 character codes - charmap += b'\0' * 0xff00 - continue - # Character set contains non-BMP character codes. - # For range, all BMP characters in the range are already - # proceeded. - if fixup: # IGNORECASE and not LOCALE - # For now, IN_UNI_IGNORE+LITERAL and - # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP - # characters, because two characters (at least one of - # which is not in the BMP) match case-insensitively - # if and only if: - # 1) c1.lower() == c2.lower() - # 2) c1.lower() == c2 or c1.lower().upper() == c2 - # Also, both c.lower() and c.lower().upper() are single - # characters for every non-BMP character. - if op is RANGE: - if fixes: # not ASCII - op = RANGE_UNI_IGNORE - hascased = True - else: - assert op is LITERAL - if not hascased and iscased(av): - hascased = True - tail.append((op, av)) - break - - # compress character map - runs = [] - q = 0 - while True: - p = charmap.find(1, q) - if p < 0: - break - if len(runs) >= 2: - runs = None - break - q = charmap.find(0, p) - if q < 0: - runs.append((p, len(charmap))) - break - runs.append((p, q)) - if runs is not None: - # use literal/range - for p, q in runs: - if q - p == 1: - out.append((LITERAL, p)) - else: - out.append((RANGE, (p, q - 1))) - out += tail - # if the case was changed or new representation is more compact - if hascased or len(out) < len(charset): - return out, hascased - # else original character set is good enough - return charset, hascased - - # use bitmap - if len(charmap) == 256: - data = _mk_bitmap(charmap) - out.append((CHARSET, data)) - out += tail - return out, hascased - - # To represent a big charset, first a bitmap of all characters in the - # set is constructed. Then, this bitmap is sliced into chunks of 256 - # characters, duplicate chunks are eliminated, and each chunk is - # given a number. In the compiled expression, the charset is - # represented by a 32-bit word sequence, consisting of one word for - # the number of different chunks, a sequence of 256 bytes (64 words) - # of chunk numbers indexed by their original chunk position, and a - # sequence of 256-bit chunks (8 words each). - - # Compression is normally good: in a typical charset, large ranges of - # Unicode will be either completely excluded (e.g. if only cyrillic - # letters are to be matched), or completely included (e.g. if large - # subranges of Kanji match). These ranges will be represented by - # chunks of all one-bits or all zero-bits. - - # Matching can be also done efficiently: the more significant byte of - # the Unicode character is an index into the chunk number, and the - # less significant byte is a bit index in the chunk (just like the - # CHARSET matching). - - charmap = charmap.take_bytes() # should be hashable - comps = {} - mapping = bytearray(256) - block = 0 - data = bytearray() - for i in range(0, 65536, 256): - chunk = charmap[i: i + 256] - if chunk in comps: - mapping[i // 256] = comps[chunk] - else: - mapping[i // 256] = comps[chunk] = block - block += 1 - data += chunk - data = _mk_bitmap(data) - data[0:0] = [block] + _bytes_to_codes(mapping) - out.append((BIGCHARSET, data)) - out += tail - return out, hascased - -_CODEBITS = _sre.CODESIZE * 8 -MAXCODE = (1 << _CODEBITS) - 1 -_BITS_TRANS = b'0' + b'1' * 255 -def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): - s = bits.translate(_BITS_TRANS)[::-1] - return [_int(s[i - _CODEBITS: i], 2) - for i in range(len(s), 0, -_CODEBITS)] - -def _bytes_to_codes(b): - # Convert block indices to word array - a = memoryview(b).cast('I') - assert a.itemsize == _sre.CODESIZE - assert len(a) * a.itemsize == len(b) - return a.tolist() - -def _simple(p): - # check if this subpattern is a "simple" operator - if len(p) != 1: - return False - op, av = p[0] - if op is SUBPATTERN: - return av[0] is None and _simple(av[-1]) - return op in _UNIT_CODES - -def _generate_overlap_table(prefix): - """ - Generate an overlap table for the following prefix. - An overlap table is a table of the same size as the prefix which - informs about the potential self-overlap for each index in the prefix: - - if overlap[i] == 0, prefix[i:] can't overlap prefix[0:...] - - if overlap[i] == k with 0 < k <= i, prefix[i-k+1:i+1] overlaps with - prefix[0:k] - """ - table = [0] * len(prefix) - for i in range(1, len(prefix)): - idx = table[i - 1] - while prefix[i] != prefix[idx]: - if idx == 0: - table[i] = 0 - break - idx = table[idx - 1] - else: - table[i] = idx + 1 - return table - -def _get_iscased(flags): - if not flags & SRE_FLAG_IGNORECASE: - return None - elif flags & SRE_FLAG_UNICODE: - return _sre.unicode_iscased - else: - return _sre.ascii_iscased - -def _get_literal_prefix(pattern, flags): - # look for literal prefix - prefix = [] - prefixappend = prefix.append - prefix_skip = None - iscased = _get_iscased(flags) - for op, av in pattern.data: - if op is LITERAL: - if iscased and iscased(av): - break - prefixappend(av) - elif op is SUBPATTERN: - group, add_flags, del_flags, p = av - flags1 = _combine_flags(flags, add_flags, del_flags) - if flags1 & SRE_FLAG_IGNORECASE and flags1 & SRE_FLAG_LOCALE: - break - prefix1, prefix_skip1, got_all = _get_literal_prefix(p, flags1) - if prefix_skip is None: - if group is not None: - prefix_skip = len(prefix) - elif prefix_skip1 is not None: - prefix_skip = len(prefix) + prefix_skip1 - prefix.extend(prefix1) - if not got_all: - break - else: - break - else: - return prefix, prefix_skip, True - return prefix, prefix_skip, False - -def _get_charset_prefix(pattern, flags): - while True: - if not pattern.data: - return None - op, av = pattern.data[0] - if op is not SUBPATTERN: - break - group, add_flags, del_flags, pattern = av - flags = _combine_flags(flags, add_flags, del_flags) - if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: - return None - - iscased = _get_iscased(flags) - if op is LITERAL: - if iscased and iscased(av): - return None - return [(op, av)] - elif op is BRANCH: - charset = [] - charsetappend = charset.append - for p in av[1]: - if not p: - return None - op, av = p[0] - if op is LITERAL and not (iscased and iscased(av)): - charsetappend((op, av)) - else: - return None - return charset - elif op is IN: - charset = av - if iscased: - for op, av in charset: - if op is LITERAL: - if iscased(av): - return None - elif op is RANGE: - if av[1] > 0xffff: - return None - if any(map(iscased, range(av[0], av[1]+1))): - return None - return charset - return None - -def _compile_info(code, pattern, flags): - # internal: compile an info block. in the current version, - # this contains min/max pattern width, and an optional literal - # prefix or a character map - lo, hi = pattern.getwidth() - if hi > MAXCODE: - hi = MAXCODE - if lo == 0: - code.extend([INFO, 4, 0, lo, hi]) - return - # look for a literal prefix - prefix = [] - prefix_skip = 0 - charset = None # not used - if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): - # look for literal prefix - prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) - # if no prefix, look for charset prefix - if not prefix: - charset = _get_charset_prefix(pattern, flags) - if charset: - charset, hascased = _optimize_charset(charset) - assert not hascased - if charset == _CHARSET_ALL: - charset = None -## if prefix: -## print("*** PREFIX", prefix, prefix_skip) -## if charset: -## print("*** CHARSET", charset) - # add an info block - emit = code.append - emit(INFO) - skip = len(code); emit(0) - # literal flag - mask = 0 - if prefix: - mask = SRE_INFO_PREFIX - if prefix_skip is None and got_all: - mask = mask | SRE_INFO_LITERAL - elif charset: - mask = mask | SRE_INFO_CHARSET - emit(mask) - # pattern length - if lo < MAXCODE: - emit(lo) - else: - emit(MAXCODE) - prefix = prefix[:MAXCODE] - emit(hi) - # add literal prefix - if prefix: - emit(len(prefix)) # length - if prefix_skip is None: - prefix_skip = len(prefix) - emit(prefix_skip) # skip - code.extend(prefix) - # generate overlap table - code.extend(_generate_overlap_table(prefix)) - elif charset: - _compile_charset(charset, flags, code) - code[skip] = len(code) - skip - def isstring(obj): return isinstance(obj, (str, bytes)) diff --git a/Lib/re/_optimizer.py b/Lib/re/_optimizer.py new file mode 100644 index 000000000000000..5e3892583a64c9b --- /dev/null +++ b/Lib/re/_optimizer.py @@ -0,0 +1,397 @@ +# +# Secret Labs' Regular Expression Engine +# +# optimizations for the compiler +# +# Copyright (c) 1997-2001 by Secret Labs AB. All rights reserved. +# +# See the __init__.py file for information on usage and redistribution. +# + +"""Internal support module for sre. + +Optimization passes used by the compiler: character-set optimization +(:func:`_optimize_charset`), the "simple" repeat-body test (:func:`_simple`), +and the literal/charset prefix info block (:func:`_compile_info`). +""" + +import _sre +from . import _parser +from ._constants import * + +_CHARSET_ALL = [(NEGATE, None)] +_UNIT_CODES = {LITERAL, NOT_LITERAL, ANY, IN, CATEGORY} + +def _combine_flags(flags, add_flags, del_flags, + TYPE_FLAGS=_parser.TYPE_FLAGS): + if add_flags & TYPE_FLAGS: + flags &= ~TYPE_FLAGS + return (flags | add_flags) & ~del_flags + +def _compile_charset(charset, flags, code): + # compile charset subprogram + emit = code.append + for op, av in charset: + emit(op) + if op is NEGATE: + pass + elif op is LITERAL: + emit(av) + elif op is RANGE or op is RANGE_UNI_IGNORE: + emit(av[0]) + emit(av[1]) + elif op is CHARSET: + code.extend(av) + elif op is BIGCHARSET: + code.extend(av) + elif op is CATEGORY: + if flags & SRE_FLAG_LOCALE: + emit(CH_LOCALE[av]) + elif flags & SRE_FLAG_UNICODE: + emit(CH_UNICODE[av]) + else: + emit(av) + else: + raise PatternError(f"internal: unsupported set operator {op!r}") + emit(FAILURE) + +def _optimize_charset(charset, iscased=None, fixup=None, fixes=None): + # internal: optimize character set + out = [] + tail = [] + charmap = bytearray(256) + hascased = False + for op, av in charset: + while True: + try: + if op is LITERAL: + if fixup: # IGNORECASE and not LOCALE + av = fixup(av) + charmap[av] = 1 + if fixes and av in fixes: + for k in fixes[av]: + charmap[k] = 1 + if not hascased and iscased(av): + hascased = True + else: + charmap[av] = 1 + elif op is RANGE: + r = range(av[0], av[1]+1) + if fixup: # IGNORECASE and not LOCALE + if fixes: + for i in map(fixup, r): + charmap[i] = 1 + if i in fixes: + for k in fixes[i]: + charmap[k] = 1 + else: + for i in map(fixup, r): + charmap[i] = 1 + if not hascased: + hascased = any(map(iscased, r)) + else: + for i in r: + charmap[i] = 1 + elif op is NEGATE: + out.append((op, av)) + elif op is CATEGORY and tail and (CATEGORY, CH_NEGATE[av]) in tail: + # Optimize [\s\S] etc. + out = [] if out else _CHARSET_ALL + return out, False + else: + tail.append((op, av)) + except IndexError: + if len(charmap) == 256: + # character set contains non-UCS1 character codes + charmap += b'\0' * 0xff00 + continue + # Character set contains non-BMP character codes. + # For range, all BMP characters in the range are already + # proceeded. + if fixup: # IGNORECASE and not LOCALE + # For now, IN_UNI_IGNORE+LITERAL and + # IN_UNI_IGNORE+RANGE_UNI_IGNORE work for all non-BMP + # characters, because two characters (at least one of + # which is not in the BMP) match case-insensitively + # if and only if: + # 1) c1.lower() == c2.lower() + # 2) c1.lower() == c2 or c1.lower().upper() == c2 + # Also, both c.lower() and c.lower().upper() are single + # characters for every non-BMP character. + if op is RANGE: + if fixes: # not ASCII + op = RANGE_UNI_IGNORE + hascased = True + else: + assert op is LITERAL + if not hascased and iscased(av): + hascased = True + tail.append((op, av)) + break + + # compress character map + runs = [] + q = 0 + while True: + p = charmap.find(1, q) + if p < 0: + break + if len(runs) >= 2: + runs = None + break + q = charmap.find(0, p) + if q < 0: + runs.append((p, len(charmap))) + break + runs.append((p, q)) + if runs is not None: + # use literal/range + for p, q in runs: + if q - p == 1: + out.append((LITERAL, p)) + else: + out.append((RANGE, (p, q - 1))) + out += tail + # if the case was changed or new representation is more compact + if hascased or len(out) < len(charset): + return out, hascased + # else original character set is good enough + return charset, hascased + + # use bitmap + if len(charmap) == 256: + data = _mk_bitmap(charmap) + out.append((CHARSET, data)) + out += tail + return out, hascased + + # To represent a big charset, first a bitmap of all characters in the + # set is constructed. Then, this bitmap is sliced into chunks of 256 + # characters, duplicate chunks are eliminated, and each chunk is + # given a number. In the compiled expression, the charset is + # represented by a 32-bit word sequence, consisting of one word for + # the number of different chunks, a sequence of 256 bytes (64 words) + # of chunk numbers indexed by their original chunk position, and a + # sequence of 256-bit chunks (8 words each). + + # Compression is normally good: in a typical charset, large ranges of + # Unicode will be either completely excluded (e.g. if only cyrillic + # letters are to be matched), or completely included (e.g. if large + # subranges of Kanji match). These ranges will be represented by + # chunks of all one-bits or all zero-bits. + + # Matching can be also done efficiently: the more significant byte of + # the Unicode character is an index into the chunk number, and the + # less significant byte is a bit index in the chunk (just like the + # CHARSET matching). + + charmap = charmap.take_bytes() # should be hashable + comps = {} + mapping = bytearray(256) + block = 0 + data = bytearray() + for i in range(0, 65536, 256): + chunk = charmap[i: i + 256] + if chunk in comps: + mapping[i // 256] = comps[chunk] + else: + mapping[i // 256] = comps[chunk] = block + block += 1 + data += chunk + data = _mk_bitmap(data) + data[0:0] = [block] + _bytes_to_codes(mapping) + out.append((BIGCHARSET, data)) + out += tail + return out, hascased + +_CODEBITS = _sre.CODESIZE * 8 +MAXCODE = (1 << _CODEBITS) - 1 +_BITS_TRANS = b'0' + b'1' * 255 +def _mk_bitmap(bits, _CODEBITS=_CODEBITS, _int=int): + s = bits.translate(_BITS_TRANS)[::-1] + return [_int(s[i - _CODEBITS: i], 2) + for i in range(len(s), 0, -_CODEBITS)] + +def _bytes_to_codes(b): + # Convert block indices to word array + a = memoryview(b).cast('I') + assert a.itemsize == _sre.CODESIZE + assert len(a) * a.itemsize == len(b) + return a.tolist() + +def _simple(p): + # check if this subpattern is a "simple" operator + if len(p) != 1: + return False + op, av = p[0] + if op is SUBPATTERN: + return av[0] is None and _simple(av[-1]) + return op in _UNIT_CODES + +def _generate_overlap_table(prefix): + """ + Generate an overlap table for the following prefix. + An overlap table is a table of the same size as the prefix which + informs about the potential self-overlap for each index in the prefix: + - if overlap[i] == 0, prefix[i:] can't overlap prefix[0:...] + - if overlap[i] == k with 0 < k <= i, prefix[i-k+1:i+1] overlaps with + prefix[0:k] + """ + table = [0] * len(prefix) + for i in range(1, len(prefix)): + idx = table[i - 1] + while prefix[i] != prefix[idx]: + if idx == 0: + table[i] = 0 + break + idx = table[idx - 1] + else: + table[i] = idx + 1 + return table + +def _get_iscased(flags): + if not flags & SRE_FLAG_IGNORECASE: + return None + elif flags & SRE_FLAG_UNICODE: + return _sre.unicode_iscased + else: + return _sre.ascii_iscased + +def _get_literal_prefix(pattern, flags): + # look for literal prefix + prefix = [] + prefixappend = prefix.append + prefix_skip = None + iscased = _get_iscased(flags) + for op, av in pattern.data: + if op is LITERAL: + if iscased and iscased(av): + break + prefixappend(av) + elif op is SUBPATTERN: + group, add_flags, del_flags, p = av + flags1 = _combine_flags(flags, add_flags, del_flags) + if flags1 & SRE_FLAG_IGNORECASE and flags1 & SRE_FLAG_LOCALE: + break + prefix1, prefix_skip1, got_all = _get_literal_prefix(p, flags1) + if prefix_skip is None: + if group is not None: + prefix_skip = len(prefix) + elif prefix_skip1 is not None: + prefix_skip = len(prefix) + prefix_skip1 + prefix.extend(prefix1) + if not got_all: + break + else: + break + else: + return prefix, prefix_skip, True + return prefix, prefix_skip, False + +def _get_charset_prefix(pattern, flags): + while True: + if not pattern.data: + return None + op, av = pattern.data[0] + if op is not SUBPATTERN: + break + group, add_flags, del_flags, pattern = av + flags = _combine_flags(flags, add_flags, del_flags) + if flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE: + return None + + iscased = _get_iscased(flags) + if op is LITERAL: + if iscased and iscased(av): + return None + return [(op, av)] + elif op is CATEGORY: + return [(op, av)] + elif op is BRANCH: + charset = [] + charsetappend = charset.append + for p in av[1]: + if not p: + return None + op, av = p[0] + if op is LITERAL and not (iscased and iscased(av)): + charsetappend((op, av)) + else: + return None + return charset + elif op is IN: + charset = av + if iscased: + for op, av in charset: + if op is LITERAL: + if iscased(av): + return None + elif op is RANGE: + if av[1] > 0xffff: + return None + if any(map(iscased, range(av[0], av[1]+1))): + return None + return charset + return None + +def _compile_info(code, pattern, flags): + # internal: compile an info block. in the current version, + # this contains min/max pattern width, and an optional literal + # prefix or a character map + lo, hi = pattern.getwidth() + if hi > MAXCODE: + hi = MAXCODE + if lo == 0: + code.extend([INFO, 4, 0, lo, hi]) + return + # look for a literal prefix + prefix = [] + prefix_skip = 0 + charset = None # not used + if not (flags & SRE_FLAG_IGNORECASE and flags & SRE_FLAG_LOCALE): + # look for literal prefix + prefix, prefix_skip, got_all = _get_literal_prefix(pattern, flags) + # if no prefix, look for charset prefix + if not prefix: + charset = _get_charset_prefix(pattern, flags) + if charset: + charset, hascased = _optimize_charset(charset) + assert not hascased + if charset == _CHARSET_ALL: + charset = None +## if prefix: +## print("*** PREFIX", prefix, prefix_skip) +## if charset: +## print("*** CHARSET", charset) + # add an info block + emit = code.append + emit(INFO) + skip = len(code); emit(0) + # literal flag + mask = 0 + if prefix: + mask = SRE_INFO_PREFIX + if prefix_skip is None and got_all: + mask = mask | SRE_INFO_LITERAL + elif charset: + mask = mask | SRE_INFO_CHARSET + emit(mask) + # pattern length + if lo < MAXCODE: + emit(lo) + else: + emit(MAXCODE) + prefix = prefix[:MAXCODE] + emit(hi) + # add literal prefix + if prefix: + emit(len(prefix)) # length + if prefix_skip is None: + prefix_skip = len(prefix) + emit(prefix_skip) # skip + code.extend(prefix) + # generate overlap table + code.extend(_generate_overlap_table(prefix)) + elif charset: + _compile_charset(charset, flags, code) + code[skip] = len(code) - skip diff --git a/Lib/re/_parser.py b/Lib/re/_parser.py index bd189fe0695f801..cc2b66c54b66811 100644 --- a/Lib/re/_parser.py +++ b/Lib/re/_parser.py @@ -27,6 +27,7 @@ _REPEATCODES = frozenset({MIN_REPEAT, MAX_REPEAT, POSSESSIVE_REPEAT}) _UNITCODES = frozenset({ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY}) +_SETITEMCODES = frozenset({LITERAL, CATEGORY}) ESCAPES = { r"\a": (LITERAL, ord("\a")), @@ -43,12 +44,12 @@ r"\A": (AT, AT_BEGINNING_STRING), # start of string r"\b": (AT, AT_BOUNDARY), r"\B": (AT, AT_NON_BOUNDARY), - r"\d": (IN, [(CATEGORY, CATEGORY_DIGIT)]), - r"\D": (IN, [(CATEGORY, CATEGORY_NOT_DIGIT)]), - r"\s": (IN, [(CATEGORY, CATEGORY_SPACE)]), - r"\S": (IN, [(CATEGORY, CATEGORY_NOT_SPACE)]), - r"\w": (IN, [(CATEGORY, CATEGORY_WORD)]), - r"\W": (IN, [(CATEGORY, CATEGORY_NOT_WORD)]), + r"\d": (CATEGORY, CATEGORY_DIGIT), + r"\D": (CATEGORY, CATEGORY_NOT_DIGIT), + r"\s": (CATEGORY, CATEGORY_SPACE), + r"\S": (CATEGORY, CATEGORY_NOT_SPACE), + r"\w": (CATEGORY, CATEGORY_WORD), + r"\W": (CATEGORY, CATEGORY_NOT_WORD), r"\z": (AT, AT_END_STRING), # end of string r"\Z": (AT, AT_END_STRING), # end of string (obsolete) } @@ -315,7 +316,7 @@ def _class_escape(source, escape): if code: return code code = CATEGORIES.get(escape) - if code and code[0] is IN: + if code and code[0] is CATEGORY: return code try: c = escape[1:2] @@ -493,7 +494,7 @@ def _parse_sub(source, state, verbose, nested): if len(item) != 1: break op, av = item[0] - if op is LITERAL: + if op in _SETITEMCODES: set.append((op, av)) elif op is IN and av[0][0] is not NEGATE: set.extend(av) @@ -508,6 +509,201 @@ def _parse_sub(source, state, verbose, nested): subpattern.append((BRANCH, (None, items))) return subpattern +def _charset_node(items): + # One element matching a character in the union `items`. A lone LITERAL or + # CATEGORY is already a one-character matcher and needs no IN wrapper. + if len(items) == 1 and items[0][0] in _SETITEMCODES: + return items[0] + return (IN, items) + +def _flat_items(elements): + # The items if `elements` is a single flat charset (no complement), else + # None -- the dual of _charset_node: a lone LITERAL or CATEGORY is an item. + if len(elements) == 1: + op, av = elements[0] + if op in _SETITEMCODES: + return [elements[0]] + if op is IN and all(o is not NEGATE for o, _av in av): + return av + return None + +def _union(left, right, state): + # A || B: merge two flat character classes into one charset where possible, + # else alternate the one-character matchers. + left_items = _flat_items(left) + right_items = _flat_items(right) + if left_items is not None and right_items is not None: + return [_charset_node(_uniq(left_items + right_items))] + return [(BRANCH, (None, [SubPattern(state, left), + SubPattern(state, right)]))] + +def _intersect(left, right, state): + # A && B: A, then require the same character to also match B (lookbehind). + return left + [(ASSERT, (-1, SubPattern(state, right)))] + +def _difference(left, right, state): + # A -- B: A, then require the character not to match B (lookbehind). + return left + [(ASSERT_NOT, (-1, SubPattern(state, right)))] + +# Map a set-operator token to the function combining the accumulated result +# with the next operand. +_SETOPS = {'||': _union, '&&': _intersect, '--': _difference} + +def _operand_elements(set, compound): + # The operand's elements: a standalone nested set, else the member union. + if compound is not None: + return compound + return [_charset_node(_uniq(set))] + +def _parse_operand(source, state, nested, here, allow_nested): + # Read one operand, stopping at a set operator or the closing ']'. An + # operand is either a union of members/ranges/escapes or, when allow_nested, + # a single nested set ([...]) -- not a mix. Return (elements, terminator), + # where terminator is the operator that ended the operand, or None at the end + # of the class. + _ord = ord + sourceget = source.get + sourcematch = source.match + set = [] + setappend = set.append + compound = None # elements of a standalone nested-set operand + if allow_nested and sourcematch("["): + # A nested set after an operator is the whole operand, used as-is (not + # wrapped in a group); it cannot be combined with loose members. + compound = _parse_charset(source, state, nested + 1) + while True: + this = sourceget() + if this is None: + raise source.error("unterminated character set", + source.tell() - here) + if set or compound is not None: + if this == "]": + return _operand_elements(set, compound), None + if this in '-&|~' and source.next == this: + if this == '~': + import warnings + warnings.warn( + 'Possible set symmetric difference at position %d' + % (source.tell() - 1), + FutureWarning, stacklevel=nested + 8 + ) + else: + # '--', '&&' or '||' ends this operand and starts the next. + sourceget() # consume the second operator character + return _operand_elements(set, compound), this + this + if this[0] == "\\": + code1 = _class_escape(source, this) + else: + code1 = LITERAL, _ord(this) + if compound is not None: + # A standalone nested set cannot be combined with other members. + raise source.error("unsupported nested set operand", + source.tell() - here) + # Past this point the operand is a plain member set (compound is None). + if sourcematch("-"): + # potential range + that = sourceget() + if that is None: + raise source.error("unterminated character set", + source.tell() - here) + if that == "]": + # A trailing '-' is a literal. + setappend(code1) + setappend((LITERAL, _ord("-"))) + return [_charset_node(_uniq(set))], None + if that == "-": + # 'X--': difference, not a range. '--' after a single member + # lands here because the range probe consumed the first '-'. + setappend(code1) + return [_charset_node(_uniq(set))], "--" + if that[0] == "\\": + code2 = _class_escape(source, that) + else: + code2 = LITERAL, _ord(that) + if code1[0] != LITERAL or code2[0] != LITERAL: + msg = "bad character range %s-%s" % (this, that) + raise source.error(msg, len(this) + 1 + len(that)) + lo = code1[1] + hi = code2[1] + if hi < lo: + msg = "bad character range %s-%s" % (this, that) + raise source.error(msg, len(this) + 1 + len(that)) + setappend((RANGE, (lo, hi))) + else: + setappend(code1) + +def _complement(elements, state): + # The complement of `elements` (a single matcher, or a set operation as a + # head followed by lookbehind assertions). De Morgan pushes the negation in + # -- recursively through nested set operations -- so no lookahead is needed. + op, av = elements[0] + if op is LITERAL: + result = [(NOT_LITERAL, av)] + elif op is NOT_LITERAL: + result = [(LITERAL, av)] + elif op is CATEGORY: + result = [(CATEGORY, CH_NEGATE[av])] + elif op is IN: + # Negate by toggling a leading NEGATE: a doubly negated set flips back + # to positive instead of stacking a second NEGATE. + if av[0][0] is NEGATE: + result = [(IN, av[1:])] + else: + result = [(IN, [(NEGATE, None)] + av)] + else: + # An un-merged union (A||B as an alternation). De Morgan: + # ~(A | B | ...) = ~A & ~B & ... -- intersect the operand complements. + assert op is BRANCH + branches = av[1] + result = _complement(branches[0].data, state) + for sub in branches[1:]: + result = _intersect(result, _complement(sub.data, state), state) + # A set operation: a head followed by lookbehind assertions. De Morgan: + # ~(head & ~B & C ...) = ~head | B | ~C ... + for op, av in elements[1:]: + if op is ASSERT_NOT: # '--' operand B: union with B + result = _union(result, av[1].data, state) + else: # '&&' operand B (ASSERT): union with [^B] + result = _union(result, _complement(av[1].data, state), state) + return result + +def _parse_charset(source, state, nested): + # Parse a character set, assuming the opening '[' has been consumed, up to + # and including the closing ']'. Return a list of subpattern elements that + # together consume exactly one character. + # + # A set operation (UTS #18 RL1.3) maps to assertions on, or alternatives of, + # the matched character: + # [A--B] -> A (? A (?<=[B]) intersection + # [A||B] -> [AB] or (?:A|B) union + # Operators chain left-to-right with no precedence. A leading '^' negates by + # De Morgan, pushing the negation into the operands (no lookahead needed): + # [^A--B] -> [^A] | B ; [^A&&B] -> [^A] | [^B] ; [^A||B] -> [^A] && [^B] + # Each operand compiles in its own flag context, so this is IGNORECASE-safe. + here = source.tell() - 1 + if source.next == '[': + # A '[' at the start of a class stays a literal (the first operand never + # needs grouping), but the position is reserved -- keep warning. + import warnings + warnings.warn( + 'Possible nested set at position %d' % source.tell(), + FutureWarning, stacklevel=nested + 7 + ) + negate = source.match("^") + result, term = _parse_operand(source, state, nested, here, False) + while term is not None: + combine = _SETOPS[term] + operand, term = _parse_operand(source, state, nested, here, True) + result = combine(result, operand, state) + if negate: + # Push the negation into the operands by De Morgan (see above). + result = _complement(result, state) + + # A single one-character matcher, or a set operation (head + assertions); + # the caller groups a multi-element result if a quantifier could follow. + return result + def _parse(source, state, verbose, nested, first=False): # parse a simple pattern subpattern = SubPattern(state) @@ -547,93 +743,15 @@ def _parse(source, state, verbose, nested, first=False): subpatternappend((LITERAL, _ord(this))) elif this == "[": - here = source.tell() - 1 - # character set - set = [] - setappend = set.append -## if sourcematch(":"): -## pass # handle character classes - if source.next == '[': - import warnings - warnings.warn( - 'Possible nested set at position %d' % source.tell(), - FutureWarning, stacklevel=nested + 6 - ) - negate = sourcematch("^") - # check remaining characters - while True: - this = sourceget() - if this is None: - raise source.error("unterminated character set", - source.tell() - here) - if this == "]" and set: - break - elif this[0] == "\\": - code1 = _class_escape(source, this) - else: - if set and this in '-&~|' and source.next == this: - import warnings - warnings.warn( - 'Possible set %s at position %d' % ( - 'difference' if this == '-' else - 'intersection' if this == '&' else - 'symmetric difference' if this == '~' else - 'union', - source.tell() - 1), - FutureWarning, stacklevel=nested + 6 - ) - code1 = LITERAL, _ord(this) - if sourcematch("-"): - # potential range - that = sourceget() - if that is None: - raise source.error("unterminated character set", - source.tell() - here) - if that == "]": - if code1[0] is IN: - code1 = code1[1][0] - setappend(code1) - setappend((LITERAL, _ord("-"))) - break - if that[0] == "\\": - code2 = _class_escape(source, that) - else: - if that == '-': - import warnings - warnings.warn( - 'Possible set difference at position %d' % ( - source.tell() - 2), - FutureWarning, stacklevel=nested + 6 - ) - code2 = LITERAL, _ord(that) - if code1[0] != LITERAL or code2[0] != LITERAL: - msg = "bad character range %s-%s" % (this, that) - raise source.error(msg, len(this) + 1 + len(that)) - lo = code1[1] - hi = code2[1] - if hi < lo: - msg = "bad character range %s-%s" % (this, that) - raise source.error(msg, len(this) + 1 + len(that)) - setappend((RANGE, (lo, hi))) - else: - if code1[0] is IN: - code1 = code1[1][0] - setappend(code1) - - set = _uniq(set) - # XXX: should move set optimization to compiler! - if _len(set) == 1 and set[0][0] is LITERAL: - # optimization - if negate: - subpatternappend((NOT_LITERAL, set[0][1])) - else: - subpatternappend(set[0]) + charset = _parse_charset(source, state, nested) + if len(charset) == 1: + code = charset[0] else: - if negate: - set.insert(0, (NEGATE, None)) - # charmap optimization can't be added here because - # global flags still are not known - subpatternappend((IN, set)) + # Wrap a multi-element set operation in a non-capturing group so + # a following quantifier (e.g. [a-z--[aeiou]]+) binds the whole + # operation, not just its trailing assertion. + code = (SUBPATTERN, (None, 0, 0, SubPattern(state, charset))) + subpatternappend(code) elif this in REPEAT_CHARS: # repeat previous item diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index e8cef29d00467f7..26fcda612567ea4 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -40,6 +40,9 @@ __all__ = ["Completer"] +# Sentinel object to distinguish "missing" from "present but None" +_MISSING = sentinel("MISSING") + class Completer: def __init__(self, namespace = None): """Create a new completer for the command line. @@ -179,14 +182,14 @@ def attr_matches(self, text): if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - if isinstance(getattr(type(thisobject), word, None), - property): - # bpo-44752: thisobject.word is a method decorated by - # `@property`. What follows applies a postfix if - # thisobject.word is callable, but know we know that - # this is not callable (because it is a property). - # Also, getattr(thisobject, word) will evaluate the - # property method, which is not desirable. + + class_attr = getattr(type(thisobject), word, None) + if isinstance( + class_attr, + (property, types.GetSetDescriptorType, types.MemberDescriptorType) + ) or (hasattr(class_attr, '__get__') and not callable(class_attr)): + # Avoid evaluating descriptors, which could run + # arbitrary code or raise exceptions. matches.append(match) continue @@ -194,14 +197,14 @@ def attr_matches(self, text): and isinstance(thisobject.__dict__.get(word), types.LazyImportType) - ): + ): value = thisobject.__dict__.get(word) else: - value = getattr(thisobject, word, None) + value = getattr(thisobject, word, _MISSING) - if value is not None: + if value is not _MISSING: matches.append(self._callable_postfix(value, match)) - else: + elif word in getattr(type(thisobject), '__slots__', ()): matches.append(match) if matches or not noprefix: break diff --git a/Lib/shlex.py b/Lib/shlex.py index 5959f52dd12639d..c7ffc918d53961c 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -317,8 +317,12 @@ def join(split_command): return ' '.join(quote(arg) for arg in split_command) -def quote(s): - """Return a shell-escaped version of the string *s*.""" +def quote(s, *, force=False): + """Return a shell-escaped version of the string *s*. + + If *force* is *True*, then *s* is unconditionally quoted, + even if it is already safe for a shell without being quoted. + """ if not s: return "''" @@ -329,8 +333,10 @@ def quote(s): safe_chars = (b'%+,-./0123456789:=@' b'ABCDEFGHIJKLMNOPQRSTUVWXYZ_' b'abcdefghijklmnopqrstuvwxyz') - # No quoting is needed if `s` is an ASCII string consisting only of `safe_chars` - if s.isascii() and not s.encode().translate(None, delete=safe_chars): + # No quoting is needed if we are not forcing quoting + # and `s` is an ASCII string consisting only of `safe_chars`. + if (not force + and s.isascii() and not s.encode().translate(None, delete=safe_chars)): return s # use single quotes, and put single quotes into double quotes diff --git a/Lib/shutil.py b/Lib/shutil.py index 45cbe4c855b462b..5095318da233146 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -885,10 +885,14 @@ def move(src, dst, copy_function=copy2): If dst already exists but is not a directory, it may be overwritten depending on os.rename() semantics. - If the destination is on our current filesystem, then rename() is used. - Otherwise, src is copied to the destination and then removed. Symlinks are - recreated under the new name if os.rename() fails because of cross - filesystem renames. + os.rename() is preferably used if the source and destination are on the + same filesystem. In case os.rename() fails due to OSError (e.g. the user + has write permission to *dst* file but not to its parent directory), + this method falls back to using *copy_function* silently. + Symlinks are also recreated under the new name if os.rename() fails + because of cross filesystem renames. + + It's recommended to use os.rename() if atomic move is strictly required. The optional `copy_function` argument is a callable that will be used to copy the source or it will be delegated to `copytree`. @@ -940,8 +944,8 @@ def move(src, dst, copy_function=copy2): return real_dst def _destinsrc(src, dst): - src = os.path.abspath(src) - dst = os.path.abspath(dst) + src = os.path.realpath(src) + dst = os.path.realpath(dst) if not src.endswith(os.path.sep): src += os.path.sep if not dst.endswith(os.path.sep): @@ -1303,12 +1307,6 @@ def unregister_unpack_format(name): """Removes the pack format from the registry.""" del _UNPACK_FORMATS[name] -def _ensure_directory(path): - """Ensure that the parent directory of `path` exists""" - dirname = os.path.dirname(path) - if not os.path.isdir(dirname): - os.makedirs(dirname) - def _unpack_zipfile(filename, extract_dir): """Unpack zip `filename` to `extract_dir` """ @@ -1638,15 +1636,3 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if _access_check(name, mode): return name return None - -def __getattr__(name): - if name == "ExecError": - import warnings - warnings._deprecated( - "shutil.ExecError", - f"{warnings._DEPRECATED_MSG}; it " - "isn't raised by any shutil function.", - remove=(3, 16) - ) - return RuntimeError - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/signal.py b/Lib/signal.py index c8cd3d4f597ca50..6387e5c35dbabac 100644 --- a/Lib/signal.py +++ b/Lib/signal.py @@ -15,7 +15,7 @@ 'Handlers', __name__, lambda name: name in ('SIG_DFL', 'SIG_IGN')) -if 'pthread_sigmask' in _globals: +if 'SIG_BLOCK' in _globals: _IntEnum._convert_( 'Sigmasks', __name__, lambda name: name in ('SIG_BLOCK', 'SIG_UNBLOCK', 'SIG_SETMASK')) diff --git a/Lib/site.py b/Lib/site.py index 52dd9648734c3ec..873c562d890a3bb 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -154,13 +154,27 @@ def _init_pathinfo(): return d -# Accumulated entry points from .start files across all site-packages -# directories. Execution is deferred until all paths in .pth files have been -# appended to sys.path. Map the .pth/.start file the data is found in to the -# data. -_pending_entrypoints = {} -_pending_syspaths = {} -_pending_importexecs = {} +# PEP 829 implementation notes. +# +# Startup information (.pth and .start file information) can be processed in +# implicit or explicit batches. Implicit batches are self-contained +# site.addsitedir() calls: they create a per-call StartupState, populate it +# from the site directory's .pth and .start files, run process() on it, and +# then throw the state away. +# +# main() needs different semantics: it accumulates state across multiple +# StartupState.addsitedir() calls (user-site plus all global site-packages) so +# that every sys.path extension is visible *before* any startup code (.start +# entry points and .pth import lines) runs. Callers can opt into the same +# behavior by creating a StartupState directly and calling its addsitedir(), +# addusersitepackages(), and addsitepackages() methods, then invoking +# process() once at the end of the batch. +# +# Here's the CRITICAL reentrancy invariant: recursive site.addsitedir() calls +# reached from a .start entry point or an exec'd .pth import line must not +# mutate the StartupState currently being processed. Reentrant calls reach +# the module-level site.addsitedir() shim, which always builds a fresh +# per-call state. def _read_pthstart_file(sitedir, name, suffix): @@ -194,13 +208,13 @@ def _read_pthstart_file(sitedir, name, suffix): return None, filename try: - # Accept BOM markers in .start and .pth files as we do in source files (Windows PowerShell - # 5.1 makes it hard to emit UTF-8 files without a BOM). + # Accept BOM markers in .start and .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM). content = raw_content.decode("utf-8-sig") except UnicodeDecodeError: _trace(f"Cannot read {filename!r} as UTF-8.") - # For .pth files only, and then only until Python 3.20, fallback to locale encoding for - # backward compatibility. + # For .pth files only, and then only until Python 3.20, fall back to + # locale encoding for backward compatibility. _warn_future_us( ".pth files decoded to locale encoding as a fallback", remove=(3, 20) @@ -214,153 +228,347 @@ def _read_pthstart_file(sitedir, name, suffix): return content.splitlines(), filename -def _read_pth_file(sitedir, name, known_paths): - """Parse a .pth file, accumulating sys.path extensions and import lines. +class StartupState: + """Per-batch accumulator for .pth and .start file processing. - Errors on individual lines do not abort processing of the rest of the - file (PEP 829). - """ - lines, filename = _read_pthstart_file(sitedir, name, ".pth") - if lines is None: - return + A StartupState collects sys.path extensions, deprecated .pth import lines, + and .start entry points read from one or more site-packages directories. + Calling process() applies them in PEP 829 order: paths are added to + sys.path first, then import lines from .pth files (skipping any with a + matching .start), then entry points from .start files. - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue + State lives entirely on the instance; there is no module-level pending + state. This is what makes the module reentrancy-safe: a site.addsitedir() + call reached recursively from an exec'd import line or a .start entry + point operates on a different StartupState than the one being processed by + the outer call. - # In Python 3.18 and 3.19, `import` lines are silently ignored. In - # Python 3.20 and beyond, issue a warning when `import` lines in .pth - # files are detected. - if line.startswith(("import ", "import\t")): - _warn_future_us( - "import lines in .pth files are silently ignored", - remove=(3, 18) - ) - _warn_future_us( - "import lines in .pth files are noisily ignored", - remove=(3, 20) - ) - _pending_importexecs.setdefault(filename, []).append(line) - continue + The internal data is intentionally private. The lower-level write + methods (_record_sitedir(), _read_pth_file(), _read_start_file()) are + private to the site module; the public surface is addsitedir(), + addusersitepackages(), addsitepackages(), and process(). + """ + __slots__ = ( + '_known_paths', + '_processed_sitedirs', + '_path_entries', + '_importexecs', + '_entrypoints', + ) + def __init__(self, known_paths=None): + """Create an independent startup state. + + *known_paths* is a set of case-normalized paths already present + on sys.path, used to avoid duplicate path entries. When None + (the default), it is initialized from the current sys.path. + + A caller-supplied set is stored by reference and mutated in place + as new paths are recorded; pass a fresh set per StartupState if + isolation across instances is required. + """ + self._known_paths = ( + _init_pathinfo() + if known_paths is None + else known_paths) + self._processed_sitedirs = set() + # The sys.path append ledger. This is a list of 2-tuples of the form + # (pthfile, path) where `pthfile` is the .pth file which is extending + # the path, and `path` is the directory to add to sys.path. Note that + # to preserve the interleaving semantics (i.e. .pth file paths are + # added after the sitedir in which the .pth file is found), `path` + # could be a sitedir, in which case `pthfile` will always be None. + self._path_entries = [] + # Both dicts map "" -> list + # of items collected from that file. Mapping by filename lets us + # cross-reference a .pth and its matching .start (PEP 829 import + # suppression rule) and lets _print_error report the source file + # when an entry fails. + self._importexecs = {} + self._entrypoints = {} + + def addsitedir(self, sitedir): + """Add a site directory and accumulate its .pth and .start startup data. + + Read the .pth and .start files in *sitedir* and record their + sys.path extensions, deprecated .pth import lines, and .start entry + points on this state. The recorded data is not applied until + process() is called. + + Typically used to batch multiple site directories before a single + process() call, so that every sys.path extension is visible before + any startup code runs. Reentrant calls reached from a .start entry + point or an exec'd .pth import line must not mutate the state + currently being processed; for those cases, use site.addsitedir() + instead, which always creates a fresh per-call state. + """ + self._addsitedir(sitedir, process_known_sitedirs=True) + + def addusersitepackages(self): + """Add the per-user site-packages directory, if enabled. + + The user site directory is added only when user site-packages are + enabled and the directory exists. Its startup data is accumulated + for later processing by process(). + """ + _trace("Processing user site-packages") + user_site = getusersitepackages() + if ENABLE_USER_SITE and os.path.isdir(user_site): + self.addsitedir(user_site) + + def addsitepackages(self, prefixes=None): + """Add global site-packages directories, if they exist. + + Site-packages directories are computed from *prefixes*, or from the + global PREFIXES when *prefixes* is None. Each directory's startup + data is accumulated for later processing by process(). + """ + _trace("Processing global site-packages") + for sitedir in getsitepackages(prefixes): + if os.path.isdir(sitedir): + self.addsitedir(sitedir) + + def _addsitedir(self, sitedir, *, process_known_sitedirs): + """Internal addsitedir() implementation with full dedup control. + + The public addsitedir() always uses process_known_sitedirs=True + (gh-149819 semantics). The module-level legacy known_paths shim + uses process_known_sitedirs=False to preserve 3.14 idempotency + (gh-75723). + """ + sitedir = self._record_sitedir( + sitedir, process_known_sitedirs=process_known_sitedirs) + if sitedir is None: + return try: - dir_, dircase = makepath(sitedir, line) - except Exception as exc: - _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) - continue - - if dircase in known_paths: - _trace(f"In {filename!r}, line {n:d}: " - f"skipping duplicate sys.path entry: {dir_}") - else: - _pending_syspaths.setdefault(filename, []).append(dir_) - known_paths.add(dircase) - - -def _read_start_file(sitedir, name): - """Parse a .start file for a list of entry point strings.""" - lines, filename = _read_pthstart_file(sitedir, name, ".start") - if lines is None: - return - - # PEP 829: the *presence* of a matching .start file disables `import` - # line processing in the matched .pth file, regardless of whether the - # .start file produced any entry points. Register the filename as a - # key now so an empty (or comment-only) .start file still suppresses. - entrypoints = _pending_entrypoints.setdefault(filename, []) - - for n, line in enumerate(lines, 1): - line = line.strip() - if not line or line.startswith("#"): - continue - # Syntax validation is deferred to entry-point execution time, - # where pkgutil.resolve_name(strict=True) enforces the - # pkg.mod:callable form. - entrypoints.append(line) - - -def _extend_syspath(): - # We've already filtered out duplicates, either in the existing sys.path - # or in all the .pth files we've seen. We've also abspath/normpath'd all - # the entries, so all that's left to do is to ensure that the path exists. - for filename, dirs in _pending_syspaths.items(): - for dir_ in dirs: - if os.path.exists(dir_): - _trace(f"Extending sys.path with {dir_} from {filename}") - sys.path.append(dir_) - else: - _print_error( - f"In {filename}: {dir_} does not exist; " - f"skipping sys.path append") - + names = os.listdir(sitedir) + except OSError: + return -def _exec_imports(): - # For all the `import` lines we've seen in .pth files, exec() them in - # order. However, if they come from a file with a matching .start, then - # we ignore these import lines. For the ones we do process, print a - # warning but only when -v was given. - for filename, imports in _pending_importexecs.items(): - name, dot, pth = filename.rpartition(".") - assert dot == "." and pth == "pth", f"Bad startup filename: {filename}" + # The following phases are defined by PEP 829. + # Phases 1-3: Read .pth files, accumulating paths and import lines. + pth_names = sorted( + name for name in names + if name.endswith(".pth") and not name.startswith(".") + ) + for name in pth_names: + self._read_pth_file(sitedir, name) + + # Phases 6-7: Discover .start files and accumulate their entry points. + # Import lines from .pth files with a matching .start file are + # discarded at flush time by _exec_imports(). + start_names = sorted( + name for name in names + if name.endswith(".start") and not name.startswith(".") + ) + for name in start_names: + self._read_start_file(sitedir, name) - if f"{name}.start" in _pending_entrypoints: - # Skip import lines in favor of entry points. - continue + def _record_sitedir(self, sitedir, *, process_known_sitedirs=True): + sitedir, sitedircase = makepath(sitedir) + # Have we already processed this sitedir? + if sitedircase in self._processed_sitedirs: + return None + # In legacy known_paths mode, a known sitedir means its startup files + # were already processed by an earlier addsitedir() call, so skip it + # to preserve idempotency (gh-75723). In explicit StartupState mode, + # known_paths only tracks sys.path entries; a sitedir may already be + # on sys.path (for example from $PYTHONPATH, gh-149819) but still need + # its .pth and .start files processed once. The separate + # _processed_sitedirs set is what lets explicit batches distinguish + # "already on sys.path" from "startup files already read". + if not process_known_sitedirs and sitedircase in self._known_paths: + return None + # Record that we've processed this sitedir. + self._processed_sitedirs.add(sitedircase) + if sitedircase not in self._known_paths: + self._known_paths.add(sitedircase) + # Add the sitedir to the sys.path extension ledger. There is no + # .pth file to record. + self._path_entries.append((None, sitedir)) + return sitedir + + def _read_pth_file(self, sitedir, name): + """Parse a .pth file, accumulating sys.path extensions and import lines. + + Errors on individual lines do not abort processing of the rest of + the file (PEP 829). Per-batch deduplication is done against + self._known_paths: any path already in it is skipped, and newly + accepted paths are added to it so that subsequent .pth files in + the same batch don't add them more than once. + """ + lines, filename = _read_pthstart_file(sitedir, name, ".pth") + if lines is None: + return + + for n, line in enumerate(lines, 1): + line = line.strip() + if not line or line.startswith("#"): + continue - _trace( - f"import lines in {filename} are deprecated, " - f"use entry points in a {name}.start file instead." - ) + # In Python 3.18 and 3.19, `import` lines are silently + # ignored. In Python 3.20 and beyond, issue a warning when + # `import` lines in .pth files are detected. + if line.startswith(("import ", "import\t")): + _warn_future_us( + "import lines in .pth files are silently ignored", + remove=(3, 18), + ) + _warn_future_us( + "import lines in .pth files are noisily ignored", + remove=(3, 20), + ) + self._importexecs.setdefault(filename, []).append(line) + continue - for line in imports: try: - _trace(f"Exec'ing from {filename}: {line}") - exec(line) + dir_, dircase = makepath(sitedir, line) except Exception as exc: - _print_error( - f"Error in import line from {filename}: {line}", exc) - - -def _execute_start_entrypoints(): - """Execute all accumulated .start file entry points. + _trace(f"Error in {filename!r}, line {n:d}: {line!r}", exc) + continue - Called after all site-packages directories have been processed so that - sys.path is fully populated before any entry point code runs. Uses - pkgutil.resolve_name(strict=True) which both validates the strict - pkg.mod:callable form and resolves the entry point in one step. - """ - for filename, entrypoints in _pending_entrypoints.items(): - for entrypoint in entrypoints: - try: - _trace(f"Executing entry point: {entrypoint} from {filename}") - callable_ = pkgutil.resolve_name(entrypoint, strict=True) - except ValueError as exc: - _print_error( - f"Invalid entry point syntax in {filename}: " - f"{entrypoint!r}", exc) + # PEP 829 dedup: skip paths already seen in this batch. + if dircase in self._known_paths: + _trace( + f"In {filename!r}, line {n:d}: " + f"skipping duplicate sys.path entry: {dir_}" + ) + else: + # Add this directory to the sys.path extension ledger, while + # also recording the .pth file it was found in. + self._path_entries.append((filename, dir_)) + self._known_paths.add(dircase) + + def _read_start_file(self, sitedir, name): + """Parse a .start file for a list of entry point strings.""" + lines, filename = _read_pthstart_file(sitedir, name, ".start") + if lines is None: + return + + # PEP 829: the *presence* of a matching .start file disables `import` + # line processing in the matched .pth file, regardless of whether this + # .start file contains any entry points. Register the filename as a + # key now so an empty (or comment-only) .start file still suppresses. + entrypoints = self._entrypoints.setdefault(filename, []) + + for n, line in enumerate(lines, 1): + line = line.strip() + if not line or line.startswith("#"): continue - except Exception as exc: - _print_error( - f"Error resolving entry point {entrypoint} " - f"from {filename}", exc) + # Syntax validation is deferred to entry point execution + # time, where pkgutil.resolve_name(strict=True) enforces the + # pkg.mod:callable form. + entrypoints.append(line) + + def process(self): + """Apply accumulated state in PEP 829 order. + + Phase order matters: all .pth path extensions are applied to + sys.path *before* any import line or .start entry point runs, so + that an entry point may live in a module reachable only via a + .pth-extended path. + """ + self._extend_syspath() + self._exec_imports() + self._execute_start_entrypoints() + + def _extend_syspath(self): + # Duplicate path-extension specifications have already been filtered + # out upstream across .pth files within this batch (via known_paths), + # and ledger entries are already abspath/normpath'd. .pth-derived + # entries (filename is not None) are existence-checked and skipped + # with an error if missing. Sitedir entries (filename is None) are + # appended unconditionally: legacy addsitedir() added the sitedir to + # sys.path before attempting to list it, so an unreadable or + # non-existent sitedir still landed on sys.path. Deferring the + # append to here preserves that contract. + for filename, dir_ in self._path_entries: + # As a backstop, known_paths may not have been seeded from sys.path + # (callers can pass an empty set), and multiple StartupState + # instances against the same sys.path don't share state, so always + # do a final anti-duplication check. + if dir_ in sys.path: continue - try: - callable_() - except Exception as exc: + if filename is None or os.path.exists(dir_): + if filename is not None: + _trace(f"Extending sys.path with {dir_} from {filename}") + sys.path.append(dir_) + else: _print_error( - f"Error in entry point {entrypoint} from {filename}", - exc) - + f"In {filename}: {dir_} does not exist; " + f"skipping sys.path append" + ) + + def _exec_imports(self): + # For each `import` line we've seen in a .pth file, exec() it in + # order, unless the .pth has a matching .start file in this same + # batch. In that case, PEP 829 says the import lines are + # suppressed in favor of the .start's entry points. + for filename, imports in self._importexecs.items(): + # Inject 'sitedir' local variable in the current frame for + # compatibility with Python 3.14. Especially, "-nspkg.pth" files + # generated by setuptools use: sys._getframe(1).f_locals['sitedir']. + sitedir = os.path.dirname(filename) + + # Given "/path/to/foo.pth", check whether "/path/to/foo.start" was + # registered in this same batch. + name, dot, pth = filename.rpartition(".") + assert dot == "." and pth == "pth", ( + f"Bad startup filename: {filename}" + ) + if f"{name}.start" in self._entrypoints: + _trace( + f"import lines in {filename} are suppressed " + f"due to matching {name}.start file." + ) + continue -def process_startup_files(): - """Flush all pending sys.path and entry points.""" - _extend_syspath() - _exec_imports() - _execute_start_entrypoints() - _pending_syspaths.clear() - _pending_importexecs.clear() - _pending_entrypoints.clear() + _trace( + f"import lines in {filename} are deprecated, " + f"use entry points in a {name}.start file instead." + ) + for line in imports: + try: + _trace(f"Exec'ing from {filename}: {line}") + exec(line) + except Exception as exc: + _print_error( + f"Error in import line from {filename}: {line}", + exc, + ) + + def _execute_start_entrypoints(self): + # Resolve each entry point string to a callable via + # pkgutil.resolve_name(strict=True), which both validates the + # required pkg.mod:callable form and performs the import in one + # step, then call it with no arguments. + for filename, entrypoints in self._entrypoints.items(): + for entrypoint in entrypoints: + try: + _trace( + f"Executing entry point: {entrypoint} from {filename}" + ) + callable_ = pkgutil.resolve_name(entrypoint, strict=True) + except ValueError as exc: + _print_error( + f"Invalid entry point syntax in {filename}: " + f"{entrypoint!r}", + exc, + ) + except Exception as exc: + _print_error( + f"Error resolving entry point {entrypoint} " + f"from {filename}", + exc, + ) + else: + try: + callable_() + except Exception as exc: + _print_error( + f"Error in entry point {entrypoint} from {filename}", + exc, + ) def addpackage(sitedir, name, known_paths): @@ -370,60 +578,31 @@ def addpackage(sitedir, name, known_paths): reset = True else: reset = False - _read_pth_file(sitedir, name, known_paths) - process_startup_files() - if reset: - known_paths = None - return known_paths + state = StartupState(known_paths) + state._read_pth_file(sitedir, name) + state.process() -def addsitedir(sitedir, known_paths=None, *, defer_processing_start_files=False): - """Add 'sitedir' argument to sys.path if missing and handle startup - files.""" - _trace(f"Adding directory: {sitedir!r}") - if known_paths is None: - known_paths = _init_pathinfo() - reset = True - else: - reset = False - sitedir, sitedircase = makepath(sitedir) - if not sitedircase in known_paths: - sys.path.append(sitedir) # Add path component - known_paths.add(sitedircase) - try: - names = os.listdir(sitedir) - except OSError: - return + return None if reset else known_paths - # The following phases are defined by PEP 829. - # Phases 1-3: Read .pth files, accumulating paths and import lines. - pth_names = sorted( - name for name in names - if name.endswith(".pth") and not name.startswith(".") - ) - for name in pth_names: - _read_pth_file(sitedir, name, known_paths) - - # Phases 6-7: Discover .start files and accumulate their entry points. - # Import lines from .pth files with a matching .start file are discarded - # at flush time by _exec_imports(). - start_names = sorted( - name for name in names - if name.endswith(".start") and not name.startswith(".") - ) - for name in start_names: - _read_start_file(sitedir, name) - - # Generally, when addsitedir() is called explicitly, we'll want to process - # all the startup file data immediately. However, when called through - # main(), we'll want to batch up all the startup file processing. main() - # will set this flag to True to defer processing. - if not defer_processing_start_files: - process_startup_files() - if reset: - known_paths = None +def addsitedir(sitedir, known_paths=None): + """Add a site directory and process its startup files. + For batched processing across multiple site directories, build a + StartupState explicitly and call StartupState.addsitedir() on it; that + defers .pth/.start processing until a single StartupState.process() call. + """ + _trace(f"Adding directory: {sitedir!r}") + if known_paths is None: + state = StartupState(_init_pathinfo()) + state.addsitedir(sitedir) + else: + # Preserve gh-75723 idempotency for legacy known_paths mode: a + # sitedir already present in known_paths is skipped, not reprocessed. + state = StartupState(known_paths) + state._addsitedir(sitedir, process_known_sitedirs=False) + state.process() return known_paths @@ -536,21 +715,20 @@ def getusersitepackages(): return USER_SITE -def addusersitepackages(known_paths, *, defer_processing_start_files=False): - """Add a per user site-package to sys.path - Each user has its own python directory with site-packages in the - home directory. - """ - # get the per user site-package path - # this call will also make sure USER_BASE and USER_SITE are set - _trace("Processing user site-packages") - user_site = getusersitepackages() +def addusersitepackages(known_paths): + """Add the per-user site-packages directory, if enabled. - if ENABLE_USER_SITE and os.path.isdir(user_site): - addsitedir(user_site, known_paths, defer_processing_start_files=defer_processing_start_files) + The user site directory is added only when user site-packages are enabled + and the directory exists. Return *known_paths*, updated with any paths + added by addsitedir(). + """ + state = StartupState(known_paths) + state.addusersitepackages() + state.process() return known_paths + def getsitepackages(prefixes=None): """Returns a list containing all global site-packages directories. @@ -590,15 +768,20 @@ def getsitepackages(prefixes=None): sitepackages.append(os.path.join(prefix, "Lib", "site-packages")) return sitepackages -def addsitepackages(known_paths, prefixes=None, *, defer_processing_start_files=False): - """Add site-packages to sys.path""" - _trace("Processing global site-packages") - for sitedir in getsitepackages(prefixes): - if os.path.isdir(sitedir): - addsitedir(sitedir, known_paths, defer_processing_start_files=defer_processing_start_files) +def addsitepackages(known_paths, prefixes=None): + """Add global site-packages directories, if they exist. + + Site-packages directories are computed from *prefixes*, or from the global + prefixes when *prefixes* is None. Return *known_paths*, updated with any + paths added by addsitedir(). + """ + state = StartupState(known_paths) + state.addsitepackages(prefixes) + state.process() return known_paths + def setquit(): """Define new builtins 'quit' and 'exit'. @@ -765,6 +948,15 @@ def write_history(): def venv(known_paths): + """Process pyvenv.cfg and add the venv site-packages, if applicable.""" + state = StartupState(known_paths) + _venv(state) + state.process() + return known_paths + + +def _venv(state): + """State-driven implementation of venv(); used by main() for batching.""" global PREFIXES, ENABLE_USER_SITE env = os.environ @@ -790,6 +982,7 @@ def venv(known_paths): if candidate_conf: virtual_conf = candidate_conf system_site = "true" + version, version_info = None, None # Issue 25185: Use UTF-8, as that's what the venv module uses when # writing the file. with open(virtual_conf, encoding='utf-8') as f: @@ -802,22 +995,53 @@ def venv(known_paths): system_site = value.lower() elif key == 'home': sys._home = value + elif key == 'version': + version = value + elif key == 'version_info': + version_info = value + + for field_name, field_value in [ + ('version',version), ('version_info',version_info) + ]: + if field_value is not None: + try: + major, minor = map(int, field_value.split(".")[:2]) + except (ValueError, AttributeError): + _warn( + f"Malformed {field_name} string in pyvenv.cfg: {field_value!r}", + RuntimeWarning, + ) + else: + if ( + major == sys.version_info.major + and minor != sys.version_info.minor + ): + _warn( + f"This virtual environment was created for Python {major}.{minor}, " + f"but the current interpreter is Python " + f"{sys.version_info.major}.{sys.version_info.minor}. " + "Consider running `python -m venv --upgrade` to update the environment.", + RuntimeWarning, + ) + break if sys.prefix != site_prefix: - _warn(f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', RuntimeWarning) + _warn( + f'Unexpected value in sys.prefix, expected {site_prefix}, got {sys.prefix}', + RuntimeWarning) if sys.exec_prefix != site_prefix: - _warn(f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', RuntimeWarning) + _warn( + f'Unexpected value in sys.exec_prefix, expected {site_prefix}, got {sys.exec_prefix}', + RuntimeWarning) - # Doing this here ensures venv takes precedence over user-site - addsitepackages(known_paths, [sys.prefix]) + # Doing this here ensures venv takes precedence over user-site. + state.addsitepackages([sys.prefix]) if system_site == "true": PREFIXES += [sys.base_prefix, sys.base_exec_prefix] else: ENABLE_USER_SITE = False - return known_paths - def execsitecustomize(): """Run custom site specific code, if available.""" @@ -868,24 +1092,25 @@ def main(): global ENABLE_USER_SITE orig_path = sys.path[:] - known_paths = removeduppaths() + removeduppaths() if orig_path != sys.path: # removeduppaths() might make sys.path absolute. # Fix __file__ of already imported modules too. abs_paths() - known_paths = venv(known_paths) + state = StartupState(set()) + _venv(state) + if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() - known_paths = addusersitepackages(known_paths, defer_processing_start_files=True) - known_paths = addsitepackages(known_paths, defer_processing_start_files=True) + + state.addusersitepackages() + state.addsitepackages() # PEP 829: flush accumulated data from all .pth and .start files. # Paths are extended first, then deprecated import lines are exec'd, # and finally .start entry points are executed — ensuring sys.path is - # fully populated before any startup code runs. process_startup_files() - # also clears the pending state so a later addsitedir() call does - # not re-apply already-processed data. - process_startup_files() + # fully populated before any startup code runs. + state.process() setquit() setcopyright() sethelper() diff --git a/Lib/socket.py b/Lib/socket.py index 34d897a82edd4fb..03c3fe88f15cfe6 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -123,7 +123,7 @@ def _intenum_converter(value, enum_klass): 10004: "The operation was interrupted.", 10009: "A bad file handle was passed.", 10013: "Permission denied.", - 10014: "A fault occurred on the network??", + 10014: "An invalid pointer was passed.", 10022: "An invalid operation was attempted.", 10024: "Too many open files.", 10035: "The socket operation would block.", @@ -181,7 +181,7 @@ def _intenum_converter(value, enum_klass): 11001: "Host not found.", 11002: "Nonauthoritative host not found.", 11003: "This is a nonrecoverable error.", - 11004: "Valid name, no data record requested type.", + 11004: "Valid name, no data record of requested type.", 11005: "QoS receivers.", 11006: "QoS senders.", 11007: "No QoS senders.", @@ -197,7 +197,7 @@ def _intenum_converter(value, enum_klass): 11017: "QoS flowspec error.", 11018: "Invalid QoS provider buffer.", 11019: "Invalid QoS filter style.", - 11020: "Invalid QoS filter style.", + 11020: "Invalid QoS filter type.", 11021: "Incorrect QoS filter count.", 11022: "Invalid QoS object length.", 11023: "Incorrect QoS flow count.", diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 8805442b69e0808..ec72c6943907175 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -87,14 +87,11 @@ def runsource(self, source, filename="", symbol="single"): def main(*args): - parser = ArgumentParser( - description="Python sqlite3 CLI", - color=True, - ) + parser = ArgumentParser(description="Python sqlite3 CLI") parser.add_argument( "filename", type=str, default=":memory:", nargs="?", help=( - "SQLite database to open (defaults to ':memory:'). " + "SQLite database to open (defaults to `:memory:`). " "A new database is created if the file does not previously exist." ), ) @@ -102,7 +99,7 @@ def main(*args): "sql", type=str, nargs="?", help=( "An SQL query to execute. " - "Any returned rows are printed to stdout." + "Any returned rows are printed to `stdout`." ), ) parser.add_argument( diff --git a/Lib/ssl.py b/Lib/ssl.py index 896db17baeb3dbd..3c0361330d7e951 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -721,10 +721,9 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, # root CA certificates for the given purpose. This may fail silently. context.load_default_certs(purpose) # OpenSSL 1.1.1 keylog file - if hasattr(context, 'keylog_filename'): - keylogfile = os.environ.get('SSLKEYLOGFILE') - if keylogfile and not sys.flags.ignore_environment: - context.keylog_filename = keylogfile + keylogfile = os.environ.get('SSLKEYLOGFILE') + if keylogfile and not sys.flags.ignore_environment: + context.keylog_filename = keylogfile return context def _create_unverified_context(protocol=None, *, cert_reqs=CERT_NONE, @@ -775,10 +774,9 @@ def _create_unverified_context(protocol=None, *, cert_reqs=CERT_NONE, # root CA certificates for the given purpose. This may fail silently. context.load_default_certs(purpose) # OpenSSL 1.1.1 keylog file - if hasattr(context, 'keylog_filename'): - keylogfile = os.environ.get('SSLKEYLOGFILE') - if keylogfile and not sys.flags.ignore_environment: - context.keylog_filename = keylogfile + keylogfile = os.environ.get('SSLKEYLOGFILE') + if keylogfile and not sys.flags.ignore_environment: + context.keylog_filename = keylogfile return context # Used by http.client if no context is explicitly passed. @@ -1055,7 +1053,12 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, notconn_pre_handshake_data = self.recv(1) except OSError as e: # EINVAL occurs for recv(1) on non-connected on unix sockets. - if e.errno not in (errno.ENOTCONN, errno.EINVAL): + if e.errno in (errno.ENOTCONN, errno.EINVAL): + pass + elif sys.platform == 'cygwin' and e.errno == errno.EAGAIN: + # EAGAIN occurs on Cygwin. + pass + else: raise notconn_pre_handshake_data = b'' self.setblocking(blocking) diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 38b655f2f7b9d2b..6fe2ec98fb40888 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -145,13 +145,13 @@ def __init__(self, returncode, cmd, output=None, stderr=None): def __str__(self): if self.returncode and self.returncode < 0: try: - return "Command '%s' died with %r." % ( + return "Command %r died with %r." % ( self.cmd, signal.Signals(-self.returncode)) except ValueError: - return "Command '%s' died with unknown signal %d." % ( + return "Command %r died with unknown signal %d." % ( self.cmd, -self.returncode) else: - return "Command '%s' returned non-zero exit status %d." % ( + return "Command %r returned non-zero exit status %d." % ( self.cmd, self.returncode) @property diff --git a/Lib/symtable.py b/Lib/symtable.py index c7152a70f5aa0b0..9238437191c00f9 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -240,41 +240,7 @@ def get_cells(self): class Class(SymbolTable): - - __methods = None - - def get_methods(self): - """Return a tuple of methods declared in the class. - """ - import warnings - typename = f'{self.__class__.__module__}.{self.__class__.__name__}' - warnings.warn(f'{typename}.get_methods() is deprecated ' - f'and will be removed in Python 3.16.', - DeprecationWarning, stacklevel=2) - - if self.__methods is None: - d = {} - - def is_local_symbol(ident): - flags = self._table.symbols.get(ident, 0) - return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL - - for st in self._table.children: - # pick the function-like symbols that are local identifiers - if is_local_symbol(st.name): - match st.type: - case _symtable.TYPE_FUNCTION: - d[st.name] = 1 - case _symtable.TYPE_TYPE_PARAMETERS: - # Get the function-def block in the annotation - # scope 'st' with the same identifier, if any. - scope_name = st.name - for c in st.children: - if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: - d[scope_name] = 1 - break - self.__methods = tuple(d) - return self.__methods + pass class Symbol: diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 47415adce04c2c8..fe55f48647e9a86 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -182,6 +182,8 @@ def joinuser(*args): _CONFIG_VARS = None # True iff _CONFIG_VARS has been fully initialized. _CONFIG_VARS_INITIALIZED = False +_config_vars_cached_prefix = None +_config_vars_cached_exec_prefix = None _USER_BASE = None @@ -437,6 +439,7 @@ def parse_config_h(fp, vars=None): import re define_rx = re.compile("#define ([A-Z][A-Za-z0-9_]+) (.*)\n") undef_rx = re.compile("/[*] #undef ([A-Z][A-Za-z0-9_]+) [*]/\n") + quoted_re = re.compile('^"(.*)"$') while True: line = fp.readline() @@ -445,6 +448,8 @@ def parse_config_h(fp, vars=None): m = define_rx.match(line) if m: n, v = m.group(1, 2) + if mq := quoted_re.match(v): + v = mq.group(1) try: if n in _ALWAYS_STR: raise ValueError @@ -597,16 +602,20 @@ def get_config_vars(*args): each argument in the configuration variable dictionary. """ global _CONFIG_VARS_INITIALIZED + global _config_vars_cached_prefix + global _config_vars_cached_exec_prefix # Avoid claiming the lock once initialization is complete. + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) if _CONFIG_VARS_INITIALIZED: # GH-126789: If sys.prefix or sys.exec_prefix were updated, invalidate the cache. - prefix = os.path.normpath(sys.prefix) - exec_prefix = os.path.normpath(sys.exec_prefix) - if _CONFIG_VARS['prefix'] != prefix or _CONFIG_VARS['exec_prefix'] != exec_prefix: + if _config_vars_cached_prefix != prefix or _config_vars_cached_exec_prefix != exec_prefix: with _CONFIG_VARS_LOCK: _CONFIG_VARS_INITIALIZED = False _init_config_vars() + _config_vars_cached_prefix = prefix + _config_vars_cached_exec_prefix = exec_prefix else: # Initialize the config_vars cache. with _CONFIG_VARS_LOCK: @@ -616,6 +625,8 @@ def get_config_vars(*args): # don't re-enter init_config_vars(). if _CONFIG_VARS is None: _init_config_vars() + _config_vars_cached_prefix = prefix + _config_vars_cached_exec_prefix = exec_prefix if args: vals = [] @@ -753,41 +764,3 @@ def get_python_version(): def _get_python_version_abi(): return _PY_VERSION_SHORT + get_config_var("abi_thread") - - -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. - """ - - import warnings - warnings.warn( - 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' - 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', - DeprecationWarning, - stacklevel=2, - ) - - import re - - _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" - _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" - - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. - - while True: - m = re.search(_findvar1_rx, s) or re.search(_findvar2_rx, s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] - else: - break - return s diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 4f47aaab9028d0f..fe75b658ebb3f82 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -256,6 +256,32 @@ def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): dst.write(buf) return +# Maximum number of bytes read in a single call when reading a member's +# extended header (a GNU long name/link or a pax header). The size of such +# a header is taken from the archive and is not trustworthy, so it is read in +# bounded chunks to avoid a huge up-front allocation when a crafted or +# truncated archive claims far more data than the file actually contains +# (gh-151497). +_EXTHEADER_READ_CHUNK = 1024 * 1024 # 1 MiB + +def _safe_read(fileobj, size): + """Read up to *size* bytes from *fileobj* in bounded chunks. + + Returns the same bytes as ``fileobj.read(size)`` would (including a short + result at end of file), but limits pre-allocation, so an + oversized size field in a crafted header cannot force a huge allocation. + """ + if size <= _EXTHEADER_READ_CHUNK: + return fileobj.read(size) + chunks = [] + while size > 0: + chunk = fileobj.read(min(size, _EXTHEADER_READ_CHUNK)) + if not chunk: + break + chunks.append(chunk) + size -= len(chunk) + return b"".join(chunks) + def _safe_print(s): encoding = getattr(sys.stdout, 'encoding', None) if encoding is not None: @@ -380,7 +406,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, except ImportError: raise CompressionError("bz2 module is not available") from None if mode == "r": - self.dbuf = b"" self.cmp = bz2.BZ2Decompressor() self.exception = OSError else: @@ -392,7 +417,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, except ImportError: raise CompressionError("lzma module is not available") from None if mode == "r": - self.dbuf = b"" self.cmp = lzma.LZMADecompressor() self.exception = lzma.LZMAError else: @@ -403,7 +427,6 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, except ImportError: raise CompressionError("compression.zstd module is not available") from None if mode == "r": - self.dbuf = b"" self.cmp = zstd.ZstdDecompressor() self.exception = zstd.ZstdError else: @@ -485,7 +508,6 @@ def _init_read_gz(self): """Initialize for reading a gzip compressed fileobj. """ self.cmp = self.zlib.decompressobj(-self.zlib.MAX_WBITS) - self.dbuf = b"" # taken from gzip.GzipFile with some alterations if self.__read(2) != b"\037\213": @@ -498,7 +520,7 @@ def _init_read_gz(self): if flag & 4: xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) + self.__read(xlen) if flag & 8: while True: s = self.__read(1) @@ -524,7 +546,9 @@ def seek(self, pos=0): if pos - self.pos >= 0: blocks, remainder = divmod(pos - self.pos, self.bufsize) for i in range(blocks): - self.read(self.bufsize) + data = self.read(self.bufsize) + if not data: + break self.read(remainder) else: raise StreamError("seeking backwards is not allowed") @@ -543,26 +567,44 @@ def _read(self, size): if self.comptype == "tar": return self.__read(size) - c = len(self.dbuf) - t = [self.dbuf] + c = 0 + t = [] while c < size: - # Skip underlying buffer to avoid unaligned double buffering. - if self.buf: - buf = self.buf - self.buf = b"" + if self.comptype == "gz": + # zlib interface is different than others. + # It returns data in unconsumed_tail. + if self.buf: + cbuf = self.buf + self.buf = b"" + else: + cbuf = self.fileobj.read(self.bufsize) + if not cbuf: + break + + try: + dbuf = self.cmp.decompress(cbuf, size - c) + self.buf = self.cmp.unconsumed_tail + except self.exception as e: + raise ReadError("invalid compressed data") from e else: - buf = self.fileobj.read(self.bufsize) - if not buf: - break - try: - buf = self.cmp.decompress(buf) - except self.exception as e: - raise ReadError("invalid compressed data") from e - t.append(buf) - c += len(buf) - t = b"".join(t) - self.dbuf = t[size:] - return t[:size] + # Other decompressors have needs_input. + # decompress() can buffer data internally. + if self.cmp.needs_input: + cbuf = self.fileobj.read(self.bufsize) + if not cbuf: + break + else: + cbuf = b"" + + try: + dbuf = self.cmp.decompress(cbuf, size - c) + except self.exception as e: + raise ReadError("invalid compressed data") from e + + t.append(dbuf) + c += len(dbuf) + + return b"".join(t) def __read(self, size): """Return size bytes from stream. If internal buffer is empty, @@ -830,16 +872,22 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + # A link member that resolves to the destination directory itself + # would replace it with a (sym)link, redirecting the destination + # for all subsequent members. + if target_path == dest_path: + raise OutsideDestinationError(member, target_path) normalized = os.path.normpath(member.linkname) if normalized != member.linkname: new_attrs['linkname'] = normalized if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) + # The symlink is created at `name` with trailing separators + # stripped, so its target is relative to the directory + # containing that path. + link_dir = os.path.dirname(name.rstrip('/' + os.sep)) + target_path = os.path.join(dest_path, link_dir, normalized) else: - target_path = os.path.join(dest_path, - member.linkname) + target_path = os.path.join(dest_path, normalized) target_path = os.path.realpath(target_path, strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: @@ -893,11 +941,14 @@ class TarInfo(object): size = 'Size in bytes.', mtime = 'Time of last modification.', chksum = 'Header checksum.', - type = ('File type. type is usually one of these constants: ' - 'REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' - 'CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE.'), + type = ('File type. type is usually one of these constants: ' + 'REGTYPE,\n' + 'AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' + 'CONTTYPE, CHRTYPE,\n' + 'BLKTYPE, GNUTYPE_SPARSE.'), linkname = ('Name of the target file name, which is only present ' - 'in TarInfo objects of type LNKTYPE and SYMTYPE.'), + 'in TarInfo\n' + 'objects of type LNKTYPE and SYMTYPE.'), uname = 'User name.', gname = 'Group name.', devmajor = 'Device major number.', @@ -905,9 +956,9 @@ class TarInfo(object): offset = 'The tar header starts here.', offset_data = "The file's data starts here.", pax_headers = ('A dictionary containing key-value pairs of an ' - 'associated pax extended header.'), + 'associated pax\n' + 'extended header.'), sparse = 'Sparse member information.', - _tarfile = None, _sparse_structs = None, _link_target = None, ) @@ -936,24 +987,6 @@ def __init__(self, name=""): self.sparse = None # sparse member information self.pax_headers = {} # pax header information - @property - def tarfile(self): - import warnings - warnings.warn( - 'The undocumented "tarfile" attribute of TarInfo objects ' - + 'is deprecated and will be removed in Python 3.16', - DeprecationWarning, stacklevel=2) - return self._tarfile - - @tarfile.setter - def tarfile(self, tarfile): - import warnings - warnings.warn( - 'The undocumented "tarfile" attribute of TarInfo objects ' - + 'is deprecated and will be removed in Python 3.16', - DeprecationWarning, stacklevel=2) - self._tarfile = tarfile - @property def path(self): 'In pax headers, "name" is called "path".' @@ -1419,7 +1452,7 @@ def _proc_gnulong(self, tarfile): """Process the blocks that hold a GNU longname or longlink member. """ - buf = tarfile.fileobj.read(self._block(self.size)) + buf = _safe_read(tarfile.fileobj, self._block(self.size)) # Fetch the next header and process it. try: @@ -1438,7 +1471,7 @@ def _proc_gnulong(self, tarfile): # Remove redundant slashes from directories. This is to be consistent # with frombuf(). if next.isdir(): - next.name = next.name.removesuffix("/") + next.name = next.name.rstrip("/") return next @@ -1475,7 +1508,7 @@ def _proc_pax(self, tarfile): POSIX.1-2008. """ # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) + buf = _safe_read(tarfile.fileobj, self._block(self.size)) # A pax header stores supplemental information for either # the following file (extended) or all following files @@ -2190,7 +2223,6 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): # Now, fill the TarInfo object with # information specific for the file. tarinfo = self.tarinfo() - tarinfo._tarfile = self # To be removed in 3.16. # Use os.stat or os.lstat, depending on if symlinks shall be resolved. if fileobj is None: @@ -2269,10 +2301,11 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If 'verbose' is False, only - the names of the members are printed. If it is True, an 'ls -l'-like - output is produced. 'members' is optional and must be a subset of the - list returned by getmembers(). + """Print a table of contents to sys.stdout. + + If 'verbose' is False, only the names of the members are printed. + If it is True, an 'ls -l'-like output is produced. 'members' is + optional and must be a subset of the list returned by getmembers(). """ # Convert tarinfo type to stat type. type2mode = {REGTYPE: stat.S_IFREG, SYMTYPE: stat.S_IFLNK, @@ -2363,10 +2396,12 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents - a non zero-size regular file, the 'fileobj' argument should be a binary file, - and tarinfo.size bytes are read from it and added to the archive. - You can create TarInfo objects directly, or by using gettarinfo(). + """Add the TarInfo object 'tarinfo' to the archive. + + If 'tarinfo' represents a non zero-size regular file, the 'fileobj' + argument should be a binary file, and tarinfo.size bytes are read + from it and added to the archive. You can create TarInfo objects + directly, or by using gettarinfo(). """ self._check("awx") @@ -2777,6 +2812,9 @@ def makelink_with_filter(self, tarinfo, targetpath, "makelink_with_filter: if filter_function is not None, " + "extraction_root must also not be None") try: + filter_function( + unfiltered.replace(name=tarinfo.name, deep=False), + extraction_root) filtered = filter_function(unfiltered, extraction_root) except _FILTER_ERRORS as cause: raise LinkFallbackError(tarinfo, unfiltered.name) from cause @@ -3067,7 +3105,7 @@ def main(): import argparse description = 'A simple command-line interface for tarfile module.' - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Verbose output') parser.add_argument('--filter', metavar='', diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 580d9f2b32544e4..115a187a8a85882 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6143,7 +6143,10 @@ def test_preload_resources(self): @only_run_in_spawn_testsuite("avoids redundant testing.") def test_mixed_startmethod(self): # Fork-based locks cannot be used with spawned process - for process_method in ["spawn", "forkserver"]: + test_methods = ["spawn"] + if "forkserver" in multiprocessing.get_all_start_methods(): + test_methods.append("forkserver") + for process_method in test_methods: queue = multiprocessing.get_context("fork").Queue() process_ctx = multiprocessing.get_context(process_method) p = process_ctx.Process(target=close_queue, args=(queue,)) @@ -6152,7 +6155,7 @@ def test_mixed_startmethod(self): p.start() # non-fork-based locks can be used with all other start methods - for queue_method in ["spawn", "forkserver"]: + for queue_method in test_methods: for process_method in multiprocessing.get_all_start_methods(): queue = multiprocessing.get_context(queue_method).Queue() process_ctx = multiprocessing.get_context(process_method) @@ -6307,6 +6310,8 @@ def test_resource_tracker_sigint(self): # Catchable signal (ignored by semaphore tracker) self.check_resource_tracker_death(signal.SIGINT, False) + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask') def test_resource_tracker_sigterm(self): # Catchable signal (ignored by semaphore tracker) self.check_resource_tracker_death(signal.SIGTERM, False) diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index a893932169a089b..8be5bf8aa4f5469 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -208,6 +208,16 @@ def rl(name): else: return None + try: + import _remote_debugging + except ImportError: + _remote_debugging = None + + def rd(name): + if _remote_debugging: + return getattr(_remote_debugging, name, None) + return None + # Try a range of "open" functions. # All of them should fail with TestHook(raise_on_events={"open"}) as hook: @@ -225,6 +235,8 @@ def rl(name): (rl("append_history_file"), 0, None), (rl("read_init_file"), testfn), (rl("read_init_file"), None), + (rd("BinaryWriter"), testfn, 1000, 0), + (rd("BinaryReader"), testfn), ]: if not fn: continue @@ -258,6 +270,8 @@ def rl(name): ("~/.history", "a") if rl("append_history_file") else None, (testfn, "r") if readline else None, ("", "r") if readline else None, + (testfn, "wb") if rd("BinaryWriter") else None, + (testfn, "rb") if rd("BinaryReader") else None, ] if i is not None ], diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 5d5b8e415f3cd21..28c3ab2605c45db 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -3757,6 +3757,7 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:30:45+00:00:90', # Time zone field out from range '2009-04-19T12:30:45-00:90:00', # Time zone field out from range '2009-04-19T12:30:45-00:00:90', # Time zone field out from range + '2020-2020', # Ambiguous 9-char date portion ] for bad_str in bad_strs: @@ -3773,6 +3774,7 @@ def test_fromisoformat_fails_datetime_valueerror(self): "2009-04-01T12:30:90", # Second out of range "2009-04-01T12:90:45", # Minute out of range "2009-04-01T25:30:45", # Hour out of range + "2009-00-01T24:00:00", # Month below range "2009-13-01T24:00:00", # Month out of range "9999-12-31T24:00:00", # Year out of range ] @@ -3801,6 +3803,32 @@ def test_fromisoformat_utc(self): self.assertIs(dt.tzinfo, timezone.utc) + def test_fromisoformat_utc_subsecond_offset(self): + # A UTC offset whose whole-second part is zero but with a non-zero + # microsecond part must be preserved, not collapsed to UTC. + for us in (1, -1, 999999, -999999): + with self.subTest(microseconds=us): + tz = timezone(timedelta(microseconds=us)) + dt = self.theclass(2020, 6, 15, 12, 34, 56, tzinfo=tz) + rt = self.theclass.fromisoformat(dt.isoformat()) + self.assertEqual(rt.utcoffset(), timedelta(microseconds=us)) + self.assertEqual(rt, dt) + self.assertIsNot(rt.tzinfo, timezone.utc) + + tz = timezone(timedelta(hours=5, minutes=30, seconds=15, + microseconds=123456)) + dt = self.theclass(2020, 6, 15, 12, 34, 56, tzinfo=tz) + rt = self.theclass.fromisoformat(dt.isoformat()) + self.assertEqual(rt.utcoffset(), tz.utcoffset(None)) + self.assertEqual(rt, dt) + + for tstr in ('2020-06-15T12:34:56+00:00', + '2020-06-15T12:34:56+00:00:00.000000', + '2020-06-15T12:34:56Z'): + with self.subTest(tstr=tstr): + self.assertIs(self.theclass.fromisoformat(tstr).tzinfo, + timezone.utc) + def test_fromisoformat_subclass(self): class DateTimeSubclass(self.theclass): pass @@ -7509,6 +7537,36 @@ def func(): self.assertEqual(out, b"a" * 8) self.assertEqual(err, b"") + @support.cpython_only + @support.subTests(("setup", "call"), [ + ("obj = _datetime.timedelta", "obj(seconds=2)"), + ("obj = _datetime.timedelta(seconds=2)", "obj.total_seconds()"), + ("obj = _datetime.date(2026, 6, 7)", "obj.isocalendar()"), + ]) + def test_static_datetime_types_outlive_collected_module(self, setup, call): + # gh-151039: This code used to crash + script = f"""if True: + import sys, gc + import _datetime + + {setup} # static C type, survives the module + del sys.modules['_datetime'] + del _datetime + sys.modules['_datetime'] = None # block re-import + gc.collect() # module object is collected + + try: + {call} # used to be a segmentation fault + except ImportError: + pass + else: + raise AssertionError("ImportError not raised") + """ + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/dtracedata/call_stack.py b/Lib/test/dtracedata/call_stack.py index ee9f3ae8d6c9f4a..11c0369d4baa045 100644 --- a/Lib/test/dtracedata/call_stack.py +++ b/Lib/test/dtracedata/call_stack.py @@ -5,16 +5,16 @@ def function_1(): def function_2(): function_1() -# CALL_FUNCTION_VAR +# CALL with positional args def function_3(dummy, dummy2): pass -# CALL_FUNCTION_KW +# CALL_KW (keyword arguments) def function_4(**dummy): return 1 return 2 # unreachable -# CALL_FUNCTION_VAR_KW +# CALL_FUNCTION_EX (unpacking) def function_5(dummy, dummy2, **dummy3): if False: return 7 diff --git a/Lib/test/dtracedata/call_stack.stp.expected b/Lib/test/dtracedata/call_stack.stp.expected index 32cf396f820629e..044eae62018f579 100644 --- a/Lib/test/dtracedata/call_stack.stp.expected +++ b/Lib/test/dtracedata/call_stack.stp.expected @@ -1,8 +1,11 @@ -function__entry:call_stack.py:start:23 function__entry:call_stack.py:function_1:1 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 function__return:call_stack.py:function_1:2 function__entry:call_stack.py:function_2:5 function__entry:call_stack.py:function_1:1 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 function__return:call_stack.py:function_1:2 function__return:call_stack.py:function_2:6 function__entry:call_stack.py:function_3:9 @@ -11,4 +14,3 @@ function__entry:call_stack.py:function_4:13 function__return:call_stack.py:function_4:14 function__entry:call_stack.py:function_5:18 function__return:call_stack.py:function_5:21 -function__return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/line.d b/Lib/test/dtracedata/line.d deleted file mode 100644 index 03f22db6fcc1a09..000000000000000 --- a/Lib/test/dtracedata/line.d +++ /dev/null @@ -1,7 +0,0 @@ -python$target:::line -/(copyinstr(arg1)=="test_line")/ -{ - printf("%d\t%s:%s:%s:%d\n", timestamp, - probename, basename(copyinstr(arg0)), - copyinstr(arg1), arg2); -} diff --git a/Lib/test/dtracedata/line.d.expected b/Lib/test/dtracedata/line.d.expected deleted file mode 100644 index 9b16ce76ee60a41..000000000000000 --- a/Lib/test/dtracedata/line.d.expected +++ /dev/null @@ -1,20 +0,0 @@ -line:line.py:test_line:2 -line:line.py:test_line:3 -line:line.py:test_line:4 -line:line.py:test_line:5 -line:line.py:test_line:6 -line:line.py:test_line:7 -line:line.py:test_line:8 -line:line.py:test_line:9 -line:line.py:test_line:10 -line:line.py:test_line:11 -line:line.py:test_line:4 -line:line.py:test_line:5 -line:line.py:test_line:6 -line:line.py:test_line:7 -line:line.py:test_line:8 -line:line.py:test_line:10 -line:line.py:test_line:11 -line:line.py:test_line:4 -line:line.py:test_line:12 -line:line.py:test_line:13 diff --git a/Lib/test/dtracedata/line.py b/Lib/test/dtracedata/line.py deleted file mode 100644 index 0930ff391f7a051..000000000000000 --- a/Lib/test/dtracedata/line.py +++ /dev/null @@ -1,17 +0,0 @@ -def test_line(): - a = 1 - print('# Preamble', a) - for i in range(2): - a = i - b = i+2 - c = i+3 - if c < 4: - a = c - d = a + b +c - print('#', a, b, c, d) - a = 1 - print('# Epilogue', a) - - -if __name__ == '__main__': - test_line() diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 45e229eb19f0f9d..64c035307e66542 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -245,7 +245,7 @@ def _create_parser(): 'buildbot workers') group.add_argument('--timeout', metavar='TIMEOUT', help='dump the traceback and exit if a test takes ' - 'more than TIMEOUT seconds; disabled if TIMEOUT ' + 'more than `TIMEOUT` seconds; disabled if `TIMEOUT` ' 'is negative or equals to zero') group.add_argument('--wait', action='store_true', help='wait for user input, e.g., allow a debugger ' @@ -261,11 +261,11 @@ def _create_parser(): group = parser.add_argument_group('Verbosity') group.add_argument('-v', '--verbose', action='count', - help='run tests in verbose mode with output to stdout') + help='run tests in verbose mode with output to `stdout`') group.add_argument('-w', '--rerun', action='store_true', help='re-run failed tests in verbose mode') group.add_argument('--verbose2', action='store_true', dest='rerun', - help='deprecated alias to --rerun') + help='deprecated alias to `--rerun`') group.add_argument('-W', '--verbose3', action='store_true', help='display test output on failure') group.add_argument('-q', '--quiet', action='store_true', @@ -295,22 +295,22 @@ def _create_parser(): more_details) group.add_argument('-m', '--match', metavar='PAT', dest='match_tests', action=FilterAction, const=True, - help='match test cases and methods with glob pattern PAT') + help='match test cases and methods with glob pattern `PAT`') group.add_argument('-i', '--ignore', metavar='PAT', dest='match_tests', action=FilterAction, const=False, - help='ignore test cases and methods with glob pattern PAT') + help='ignore test cases and methods with glob pattern `PAT`') group.add_argument('--matchfile', metavar='FILENAME', dest='match_tests', action=FromFileFilterAction, const=True, - help='similar to --match but get patterns from a ' + help='similar to `--match` but get patterns from a ' 'text file, one pattern per line') group.add_argument('--ignorefile', metavar='FILENAME', dest='match_tests', action=FromFileFilterAction, const=False, - help='similar to --matchfile but it receives patterns ' + help='similar to `--matchfile` but it receives patterns ' 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', - help='fail as soon as a test fails (only with -v or -W)') + help='fail as soon as a test fails (only with `-v` or `-W`)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', action='extend', type=resources_list, help='specify which special resource intensive tests ' @@ -325,7 +325,7 @@ def _create_parser(): group = parser.add_argument_group('Special runs') group.add_argument('-L', '--runleaks', action='store_true', - help='run the leaks(1) command just before exit.' + + help='run the `leaks(1)` command just before exit.' + more_details) group.add_argument('-R', '--huntrleaks', metavar='RUNCOUNTS', type=huntrleaks, @@ -333,20 +333,20 @@ def _create_parser(): 'very slow).' + more_details) group.add_argument('-j', '--multiprocess', metavar='PROCESSES', dest='use_mp', type=int, - help='run PROCESSES processes at once') + help='run `PROCESSES` processes at once') group.add_argument('--single-process', action='store_true', dest='single_process', help='always run all tests sequentially in ' - 'a single process, ignore -jN option, ' + 'a single process, ignore `-jN` option, ' 'and failed tests are also rerun sequentially ' 'in the same process') group.add_argument('--parallel-threads', metavar='PARALLEL_THREADS', type=int, - help='run copies of each test in PARALLEL_THREADS at ' + help='run copies of each test in `PARALLEL_THREADS` at ' 'once') group.add_argument('-T', '--coverage', action='store_true', dest='trace', - help='turn on code coverage tracing using the trace ' + help='turn on code coverage tracing using the `trace` ' 'module') group.add_argument('-D', '--coverdir', metavar='DIR', type=relative_filename, @@ -356,18 +356,18 @@ def _create_parser(): help='put coverage files alongside modules') group.add_argument('-t', '--threshold', metavar='THRESHOLD', type=int, - help='call gc.set_threshold(THRESHOLD)') + help='call `gc.set_threshold(THRESHOLD)`') group.add_argument('-n', '--nowindows', action='store_true', help='suppress error message boxes on Windows') group.add_argument('-F', '--forever', action='store_true', help='run the specified tests in a loop, until an ' - 'error happens; imply --failfast') + 'error happens; imply `--failfast`') group.add_argument('--list-tests', action='store_true', help="only write the name of tests that will be run, " "don't execute them") group.add_argument('--list-cases', action='store_true', - help='only write the name of test cases that will be run' - ' , don\'t execute them') + help='only write the name of test cases that will be run, ' + 'don\'t execute them') group.add_argument('-P', '--pgo', dest='pgo', action='store_true', help='enable Profile Guided Optimization (PGO) training') group.add_argument('--pgo-extended', action='store_true', @@ -390,11 +390,11 @@ def _create_parser(): group.add_argument('--tempdir', metavar='PATH', help='override the working directory for the test run') group.add_argument('--cleanup', action='store_true', - help='remove old test_python_* directories') + help='remove old `test_python_*` directories') group.add_argument('--bisect', action='store_true', - help='if some tests fail, run test.bisect_cmd on them') + help='if some tests fail, run `test.bisect_cmd` on them') group.add_argument('--pythoninfo', action='store_true', - help="run python -m test.pythoninfo before tests") + help="run `python -m test.pythoninfo` before tests") group.add_argument('--dont-add-python-opts', dest='_add_python_opts', action='store_false', help="internal option, don't use it") diff --git a/Lib/test/libregrtest/logger.py b/Lib/test/libregrtest/logger.py index fa1d4d575c8fd4f..4e011ef06f8a913 100644 --- a/Lib/test/libregrtest/logger.py +++ b/Lib/test/libregrtest/logger.py @@ -1,5 +1,6 @@ import os import time +from typing import Callable from test.support import MS_WINDOWS from .results import TestResults @@ -19,16 +20,27 @@ def __init__(self, results: TestResults, quiet: bool, pgo: bool): self._results: TestResults = results self._quiet: bool = quiet self._pgo: bool = pgo + self.get_mem_usage: Callable[[], int | None] | None = None def log(self, line: str = '') -> None: empty = not line - # add the system load prefix: "load avg: 1.80 " + # Add the memory usage: "mem: 1 GiB " + if self.get_mem_usage is not None: + mem = self.get_mem_usage() + if mem: + mib = mem / (1024*1024) + if mib >= 1024: + line = f"mem: {mib / 1024:.1f} GiB {line}" + else: + line = f"mem: {mib:.1f} MiB {line}" + + # Add the system load prefix: "load avg: 1.80 " load_avg = self.get_load_avg() if load_avg is not None: line = f"load avg: {load_avg:.2f} {line}" - # add the timestamp prefix: "0:01:05 " + # Add the timestamp prefix: "0:01:05 " log_time = time.perf_counter() - self.start_time mins, secs = divmod(int(log_time), 60) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index ac82541059cc628..8773e9df73263b7 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -27,7 +27,7 @@ printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, is_cross_compiled, get_host_runner, display_title, - EXIT_TIMEOUT) + get_process_memory_usage, EXIT_TIMEOUT) class Regrtest: @@ -393,7 +393,12 @@ def run_test( return result + def _get_mem_usage(self): + return get_process_memory_usage(os.getpid()) + def run_tests_sequentially(self, runtests: RunTests) -> None: + if not self.pgo: + self.logger.get_mem_usage = self._get_mem_usage if self.coverage: tracer = trace.Trace(trace=False, count=True) else: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 424085a0050eb59..befdac7ee77f107 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -22,7 +22,7 @@ from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, - format_duration, print_warning, count, plural) + format_duration, print_warning, count, plural, get_process_memory_usage) from .worker import create_worker_process, USE_PROCESS_GROUP if MS_WINDOWS: @@ -452,6 +452,12 @@ def wait_stopped(self, start_time: float) -> None: print_warning(f"Failed to join {self} in {format_duration(dt)}") break + def get_mem_usage(self): + popen = self._popen + if popen is None: + return + return get_process_memory_usage(popen.pid) + def get_running(workers: list[WorkerThread]) -> str | None: running: list[str] = [] @@ -473,6 +479,7 @@ def __init__(self, num_workers: int, runtests: RunTests, logger: Logger, results: TestResults) -> None: self.num_workers = num_workers self.runtests = runtests + self.logger = logger self.log = logger.log self.display_progress = logger.display_progress self.results: TestResults = results @@ -598,9 +605,21 @@ def _process_result(self, item: QueueOutput) -> TestResult: return result + def get_mem_usage(self): + usage = 0 + main_mem = get_process_memory_usage(os.getpid()) + if main_mem: + usage += main_mem + for worker in self.workers: + worker_mem = worker.get_mem_usage() + if worker_mem: + usage += worker_mem + return usage + def run(self) -> None: fail_fast = self.runtests.fail_fast fail_env_changed = self.runtests.fail_env_changed + self.logger.get_mem_usage = self.get_mem_usage self.start_workers() @@ -625,3 +644,4 @@ def run(self) -> None: # worker when we exit this function self.pending.stop() self.stop_workers() + self.logger.get_mem_usage = None diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 958a915626ad241..d0759d2626989d6 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -145,7 +145,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Storage of uncollectable GC objects (gc.garbage) -GC_GARBAGE = [] +GC_GARBAGE: list[object] = [] def _load_run_test(result: TestResult, runtests: RunTests) -> None: diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index f1f8c8bde920ae7..bacfe5e21ba0b7d 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -29,6 +29,7 @@ 'test_threadsignals', 'test_weakref', 'test_free_threading', + 'test_embed', ] # Tests that should be run with `--parallel-threads=N` under TSAN. These tests diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 00703d6c074855b..32f02429ff33076 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -12,7 +12,17 @@ import sysconfig import tempfile import textwrap +import types from collections.abc import Callable +_winapi: types.ModuleType | None +try: + import _winapi +except ImportError: + _winapi = None +try: + from _testcapi import get_process_memory_usage as _get_process_memory_usage +except ImportError: + _get_process_memory_usage = None from test import support from test.support import os_helper @@ -752,3 +762,55 @@ def display_title(title): print(title) print("#" * len(title)) print(flush=True) + + +def _get_process_memory_usage_linux(pid: int) -> int | None: + # Linux implementation: read the private memory in bytes from + # /proc/pid/smaps. + try: + fp = open(f"/proc/{pid}/smaps", "rb") + except OSError: + return None + + try: + total = 0 + with fp: + for line in fp: + # Include both Private_Clean and Private_Dirty sections. + line = line.rstrip() + if line.startswith(b"Private_") and line.endswith(b'kB'): + parts = line.split() + total += int(parts[1]) * 1024 + return total + except ProcessLookupError: + return None + + +def _get_process_memory_usage_windows(pid: int) -> int | None: + assert _winapi is not None # to make mypy happy + try: + handle = _winapi.OpenProcess(_winapi.PROCESS_QUERY_LIMITED_INFORMATION, + False, pid) + except OSError: + return None + try: + mem_info = _winapi.GetProcessMemoryInfo(handle) + finally: + _winapi.CloseHandle(handle) + return mem_info['WorkingSetSize'] + + +if _get_process_memory_usage is not None: + def get_process_memory_usage(pid: int) -> int | None: + try: + return _get_process_memory_usage(pid) + except ProcessLookupError: + return None +elif _winapi is not None: + get_process_memory_usage = _get_process_memory_usage_windows +elif sys.platform == 'linux': + get_process_memory_usage = _get_process_memory_usage_linux +else: + def get_process_memory_usage(pid: int) -> int | None: + return None +get_process_memory_usage.__doc__ = "Get process memory usage in bytes." diff --git a/Lib/test/mathdata/math_testcases.txt b/Lib/test/mathdata/math_testcases.txt index 958518824376f82..4a38a3666bab208 100644 --- a/Lib/test/mathdata/math_testcases.txt +++ b/Lib/test/mathdata/math_testcases.txt @@ -631,3 +631,1619 @@ log20646 log2 2.535995592365391e+208 -> 692.30359597460460 log20647 log2 6.2011236566089916e+233 -> 776.64177576730913 log20648 log2 2.1843274820677632e+253 -> 841.57499717289647 log20649 log2 8.7493931063474791e+297 -> 989.74182713073981 + +------------ +-- acospi -- +------------ + +acospi10000 acospi 0.0 -> 0.5 +acospi10001 acospi -0.0 -> 0.5 +acospi10002 acospi 1.0 -> 0.0 +acospi10003 acospi -1.0 -> 1.0 +acospi10004 acospi 0.5 -> 0.3333333333333333 +acospi10005 acospi -0.5 -> 0.6666666666666666 +acospi10006 acospi 0.75 -> 0.23005345616261588 +acospi10007 acospi 2.0000001329864406e-17 -> 0.5 +acospi10008 acospi 1.999999967550318e-17 -> 0.5 +acospi10009 acospi 2e-17 -> 0.5 +acospi10010 acospi 1.9999999999999998e-17 -> 0.5 +acospi10011 acospi 0.0625 -> 0.48009265723071126 +acospi10012 acospi 0.9999999403953552 -> 0.00010990189460803536 +acospi10013 acospi -0.9999999403953552 -> 0.999890098105392 +acospi10014 acospi 1.0 -> 0.0 +acospi10015 acospi 0.9999999403953552 -> 0.00010990189460803536 +acospi10016 acospi 0.9999999998835847 -> 4.857023409833964e-06 +acospi10017 acospi -0.9999999403953552 -> 0.999890098105392 +acospi10018 acospi -1.0 -> 1.0 +acospi10019 acospi -0.9999999998835847 -> 0.9999951429765902 +acospi10020 acospi 1.0 -> 0.0 +acospi10021 acospi 0.9999999403953552 -> 0.00010990189460803536 +acospi10022 acospi 0.9999999999999964 -> 2.6831517105016303e-08 +acospi10023 acospi -0.9999999403953552 -> 0.999890098105392 +acospi10024 acospi -1.0 -> 1.0 +acospi10025 acospi -0.9999999999999964 -> 0.9999999731684829 +acospi10026 acospi 1.0 -> 0.0 +acospi10027 acospi 0.9999999403953552 -> 0.00010990189460803536 +acospi10028 acospi 0.9999999999999999 -> 4.743186923619966e-09 +acospi10029 acospi -0.9999999403953552 -> 0.999890098105392 +acospi10030 acospi -1.0 -> 1.0 +acospi10031 acospi -0.9999999999999999 -> 0.999999995256813 +acospi10032 acospi 1.0 -> 0.0 +acospi10033 acospi 0.9999999403953552 -> 0.00010990189460803536 +acospi10034 acospi 0.9999999999999999 -> 4.743186923619966e-09 +acospi10035 acospi -0.9999999403953552 -> 0.999890098105392 +acospi10036 acospi -1.0 -> 1.0 +acospi10037 acospi -0.9999999999999999 -> 0.999999995256813 +acospi10038 acospi 1.0 -> 0.0 +acospi10039 acospi 0.9999999403953552 -> 0.00010990189460803536 +acospi10040 acospi 0.9999999999999999 -> 4.743186923619966e-09 +acospi10041 acospi -0.9999999403953552 -> 0.999890098105392 +acospi10042 acospi -1.0 -> 1.0 +acospi10043 acospi -0.9999999999999999 -> 0.999999995256813 +acospi10044 acospi 1.0 -> 0.0 +acospi10045 acospi 0.9999999403953552 -> 0.00010990189460803536 +acospi10046 acospi 0.9999999999999999 -> 4.743186923619966e-09 +acospi10047 acospi -0.9999999403953552 -> 0.999890098105392 +acospi10048 acospi -1.0 -> 1.0 +acospi10049 acospi -0.9999999999999999 -> 0.999999995256813 +acospi10050 acospi 0.03125 -> 0.4900511963370614 +acospi10051 acospi 0.0009765625 -> 0.49968915045236545 +acospi10052 acospi 3.0517578125e-05 -> 0.4999902859531789 +acospi10053 acospi 9.5367431640625e-07 -> 0.49999969643603687 +acospi10054 acospi 2.9802322387695312e-08 -> 0.49999999051362615 +acospi10055 acospi 9.313225746154785e-10 -> 0.4999999997035508 +acospi10056 acospi 2.9103830456733704e-11 -> 0.49999999999073597 +acospi10057 acospi 9.094947017729282e-13 -> 0.4999999999997105 +acospi10058 acospi 2.842170943040401e-14 -> 0.49999999999999095 +acospi10059 acospi 8.881784197001252e-16 -> 0.4999999999999997 +acospi10060 acospi 2.7755575615628914e-17 -> 0.5 +acospi10061 acospi 8.673617379884035e-19 -> 0.5 +acospi10062 acospi 2.710505431213761e-20 -> 0.5 +acospi10063 acospi 8.470329472543003e-22 -> 0.5 +acospi10064 acospi 2.6469779601696886e-23 -> 0.5 +acospi10065 acospi 8.271806125530277e-25 -> 0.5 +acospi10066 acospi 2.5849394142282115e-26 -> 0.5 +acospi10067 acospi 8.077935669463161e-28 -> 0.5 +acospi10068 acospi 2.524354896707238e-29 -> 0.5 +acospi10069 acospi 7.888609052210118e-31 -> 0.5 +acospi10070 acospi 2.465190328815662e-32 -> 0.5 +acospi10071 acospi 7.703719777548943e-34 -> 0.5 +acospi10072 acospi 2.407412430484045e-35 -> 0.5 +acospi10073 acospi 7.52316384526264e-37 -> 0.5 +acospi10074 acospi -0.03125 -> 0.5099488036629386 +acospi10075 acospi -2.9802322387695312e-08 -> 0.5000000094863738 +acospi10076 acospi -2.842170943040401e-14 -> 0.500000000000009 +acospi10077 acospi -2.710505431213761e-20 -> 0.5 +acospi10078 acospi -2.5849394142282115e-26 -> 0.5 +acospi10079 acospi -2.465190328815662e-32 -> 0.5 +acospi10080 acospi -2.350988701644575e-38 -> 0.5 +acospi10081 acospi -0.1251312643289566 -> 0.5399352010629168 +acospi10082 acospi 0.9386433362960815 -> 0.1120835750337698 +acospi10083 acospi 0.3664908707141876 -> 0.3805590984950769 +acospi10084 acospi 0.36649084091186523 -> 0.38055910869085624 +acospi10085 acospi 0.366490841000206 -> 0.38055910866063364 +acospi10086 acospi -0.13816863298416138 -> 0.5441215931223088 +acospi10087 acospi -0.13816864788532257 -> 0.5441215979114296 +acospi10088 acospi -0.13816864007172486 -> 0.5441215954001982 +acospi10089 acospi 0.9126854538917542 -> 0.1340047775091756 +acospi10090 acospi 0.9126853942871094 -> 0.13400482393558352 +acospi10091 acospi 0.912685444573738 -> 0.13400478476703412 +acospi10092 acospi 0.9126854445737379 -> 0.1340047847670342 +acospi10093 acospi 0.880692720413208 -> 0.1570775606195134 +acospi10094 acospi 0.8806926608085632 -> 0.15707760067276577 +acospi10095 acospi 0.8806926814412734 -> 0.15707758680795558 +acospi10096 acospi 0.8806926814412733 -> 0.15707758680795567 +acospi10097 acospi 0.2435227781534195 -> 0.4216969371828132 +acospi10098 acospi 0.2435227632522583 -> 0.42169694207322506 +acospi10099 acospi 0.24352277005396675 -> 0.42169693984097245 +acospi10100 acospi 0.1279129981994629 -> 0.45917217189807447 +acospi10101 acospi -0.11572346091270447 -> 0.5369186384114879 +acospi10102 acospi 0.9999996423721313 -> 0.0002692035702405639 +acospi10103 acospi 0.9999995827674866 -> 0.000290773090413406 +acospi10104 acospi 0.9999996241289419 -> 0.0002759844303031754 +acospi10105 acospi 0.9995118975639343 -> 0.009945766966749 +acospi10106 acospi 0.9995118379592896 -> 0.009946374261451639 +acospi10107 acospi 0.9995118644629029 -> 0.009946104228281858 +acospi10108 acospi 0.9995118644629027 -> 0.009946104228282989 +acospi10109 acospi 1.1754943508222875e-38 -> 0.5 +acospi10110 acospi 2.2250738585072014e-308 -> 0.5 +acospi10111 acospi 2.004168360008973e-292 -> 0.5 +acospi10112 acospi -1.1754943508222875e-38 -> 0.5 +acospi10113 acospi -2.2250738585072014e-308 -> 0.5 +acospi10114 acospi -2.004168360008973e-292 -> 0.5 +acospi10115 acospi 1.401298464324817e-45 -> 0.5 +acospi10116 acospi 5e-324 -> 0.5 +acospi10117 acospi -1.401298464324817e-45 -> 0.5 +acospi10118 acospi -5e-324 -> 0.5 +acospi10119 acospi -0.5356916189193726 -> 0.6799490414927629 +acospi10120 acospi 0.9258930683135986 -> 0.12331439084862783 +acospi10121 acospi 0.9258930087089539 -> 0.12331444106954574 +acospi10122 acospi 0.9258930140454763 -> 0.12331443657316782 +acospi10123 acospi 0.9997028112411499 -> 0.007760542300314392 +acospi10124 acospi 0.9997027516365051 -> 0.007761320533135704 +acospi10125 acospi 0.9997027756274229 -> 0.00776100730317654 +acospi10126 acospi 0.9997027756274228 -> 0.007761007303177989 +acospi10127 acospi 0.7071993947029114 -> 0.24995830653935586 +acospi10128 acospi 0.7071993350982666 -> 0.249958333374387 +acospi10129 acospi 0.7071993911300235 -> 0.24995830814793119 +acospi10130 acospi 0.7071993911300234 -> 0.24995830814793124 + +acospi20001 acospi inf -> nan invalid +acospi20002 acospi -inf -> nan invalid +acospi20003 acospi nan -> nan + +------------ +-- asinpi -- +------------ + +asinpi10000 asinpi 0.0 -> 0.0 +asinpi10001 asinpi -0.0 -> -0.0 +asinpi10002 asinpi 0.5 -> 0.16666666666666666 +asinpi10003 asinpi -0.5 -> -0.16666666666666666 +asinpi10004 asinpi 1.0 -> 0.5 +asinpi10005 asinpi -1.0 -> -0.5 +asinpi10006 asinpi 0.75 -> 0.2699465438373841 +asinpi10007 asinpi 0.9999999403953552 -> 0.499890098105392 +asinpi10008 asinpi -0.9999999403953552 -> -0.499890098105392 +asinpi10009 asinpi 1.0 -> 0.5 +asinpi10010 asinpi 0.9999999403953552 -> 0.499890098105392 +asinpi10011 asinpi 0.9999999998835847 -> 0.49999514297659015 +asinpi10012 asinpi -0.9999999403953552 -> -0.499890098105392 +asinpi10013 asinpi -1.0 -> -0.5 +asinpi10014 asinpi -0.9999999998835847 -> -0.49999514297659015 +asinpi10015 asinpi 1.0 -> 0.5 +asinpi10016 asinpi 0.9999999403953552 -> 0.499890098105392 +asinpi10017 asinpi 0.9999999999999964 -> 0.4999999731684829 +asinpi10018 asinpi -0.9999999403953552 -> -0.499890098105392 +asinpi10019 asinpi -1.0 -> -0.5 +asinpi10020 asinpi -0.9999999999999964 -> -0.4999999731684829 +asinpi10021 asinpi 1.0 -> 0.5 +asinpi10022 asinpi 0.9999999403953552 -> 0.499890098105392 +asinpi10023 asinpi 0.9999999999999999 -> 0.4999999952568131 +asinpi10024 asinpi -0.9999999403953552 -> -0.499890098105392 +asinpi10025 asinpi -1.0 -> -0.5 +asinpi10026 asinpi -0.9999999999999999 -> -0.4999999952568131 +asinpi10027 asinpi 1.0 -> 0.5 +asinpi10028 asinpi 0.9999999403953552 -> 0.499890098105392 +asinpi10029 asinpi 0.9999999999999999 -> 0.4999999952568131 +asinpi10030 asinpi -0.9999999403953552 -> -0.499890098105392 +asinpi10031 asinpi -1.0 -> -0.5 +asinpi10032 asinpi -0.9999999999999999 -> -0.4999999952568131 +asinpi10033 asinpi 1.0 -> 0.5 +asinpi10034 asinpi 0.9999999403953552 -> 0.499890098105392 +asinpi10035 asinpi 0.9999999999999999 -> 0.4999999952568131 +asinpi10036 asinpi -0.9999999403953552 -> -0.499890098105392 +asinpi10037 asinpi -1.0 -> -0.5 +asinpi10038 asinpi -0.9999999999999999 -> -0.4999999952568131 +asinpi10039 asinpi 1.0 -> 0.5 +asinpi10040 asinpi 0.9999999403953552 -> 0.499890098105392 +asinpi10041 asinpi 0.9999999999999999 -> 0.4999999952568131 +asinpi10042 asinpi -0.9999999403953552 -> -0.499890098105392 +asinpi10043 asinpi -1.0 -> -0.5 +asinpi10044 asinpi -0.9999999999999999 -> -0.4999999952568131 +asinpi10045 asinpi 0.03125 -> 0.009948803662938608 +asinpi10046 asinpi 0.0009765625 -> 0.0003108495476345764 +asinpi10047 asinpi 3.0517578125e-05 -> 9.714046821081509e-06 +asinpi10048 asinpi 9.5367431640625e-07 -> 3.0356396311172383e-07 +asinpi10049 asinpi 2.9802322387695312e-08 -> 9.486373847239932e-09 +asinpi10050 asinpi 9.313225746154785e-10 -> 2.964491827262479e-10 +asinpi10051 asinpi 2.9103830456733704e-11 -> 9.264036960195246e-12 +asinpi10052 asinpi 9.094947017729282e-13 -> 2.8950115500610145e-13 +asinpi10053 asinpi 2.842170943040401e-14 -> 9.04691109394067e-15 +asinpi10054 asinpi 8.881784197001252e-16 -> 2.8271597168564594e-16 +asinpi10055 asinpi 2.7755575615628914e-17 -> 8.834874115176436e-18 +asinpi10056 asinpi 8.673617379884035e-19 -> 2.760898160992636e-19 +asinpi10057 asinpi 7.888609052210118e-31 -> 2.5110222495574236e-31 +asinpi10058 asinpi 1.401298464324817e-45 -> 4.460471546887532e-46 +asinpi10059 asinpi 0.0 -> 0.0 +asinpi10060 asinpi 2.409919865102884e-181 -> 7.671013179729553e-182 +asinpi10061 asinpi 1.401298464324817e-45 -> 4.460471546887532e-46 +asinpi10062 asinpi 0.0 -> 0.0 +asinpi10063 asinpi 5e-324 -> 0.0 +asinpi10064 asinpi -0.13099800050258636 -> -0.04181814856647883 +asinpi10065 asinpi -0.21591897308826447 -> -0.06927470561424247 +asinpi10066 asinpi -0.1944168210029602 -> -0.06228143288596484 +asinpi10067 asinpi -0.1944168359041214 -> -0.062281437721417 +asinpi10068 asinpi -0.19441682493723628 -> -0.06228143416264412 +asinpi10069 asinpi 0.9866708517074585 -> 0.4479704696000938 +asinpi10070 asinpi 0.9866707921028137 -> 0.4479703530089429 +asinpi10071 asinpi 0.9866708504122479 -> 0.4479704670665621 +asinpi10072 asinpi 0.9866708504122478 -> 0.44797046706656185 +asinpi10073 asinpi -0.6866101622581482 -> -0.24090205124758424 +asinpi10074 asinpi 0.8438399434089661 -> 0.3197103200989401 +asinpi10075 asinpi -0.13135038316249847 -> -0.04193129309542388 +asinpi10076 asinpi -0.5068531036376953 -> -0.16919134974732694 +asinpi10077 asinpi -0.5068531632423401 -> -0.1691913717566472 +asinpi10078 asinpi -0.5068531437412152 -> -0.16919136455575687 +asinpi10079 asinpi -0.7008922100067139 -> -0.2472146113103207 +asinpi10080 asinpi 0.12256252765655518 -> 0.03911120286288364 +asinpi10081 asinpi 0.12256252020597458 -> 0.03911120047327442 +asinpi10082 asinpi 0.12256252763528681 -> 0.03911120285606228 +asinpi10083 asinpi 1.1754943508222875e-38 -> 3.7417147301993126e-39 +asinpi10084 asinpi 2.2250738585072014e-308 -> 7.082630066519554e-309 +asinpi10085 asinpi 2.004168360008973e-292 -> 6.379466025676106e-293 +asinpi10086 asinpi -1.1754943508222875e-38 -> -3.7417147301993126e-39 +asinpi10087 asinpi -2.2250738585072014e-308 -> -7.082630066519554e-309 +asinpi10088 asinpi -2.004168360008973e-292 -> -6.379466025676106e-293 +asinpi10089 asinpi 1.401298464324817e-45 -> 4.460471546887532e-46 +asinpi10090 asinpi 5e-324 -> 0.0 +asinpi10091 asinpi -1.401298464324817e-45 -> -4.460471546887532e-46 +asinpi10092 asinpi -5e-324 -> -0.0 +asinpi10093 asinpi 0.972168505191803 -> 0.4247258559057922 +asinpi10094 asinpi -0.19141583144664764 -> -0.06130790022106249 +asinpi10095 asinpi -0.19141584634780884 -> -0.06130790505360787 +asinpi10096 asinpi -0.19141583181687138 -> -0.06130790034112851 +asinpi10097 asinpi 0.505088746547699 -> 0.16854024296386302 +asinpi10098 asinpi 0.5050886869430542 -> 0.16854022098093674 +asinpi10099 asinpi 0.5050886987499023 -> 0.1685402253354475 +asinpi10100 asinpi 0.5050886987499021 -> 0.16854022533544744 +asinpi10101 asinpi -0.6554324626922607 -> -0.2275136094679927 +asinpi10102 asinpi -0.6554325222969055 -> -0.2275136345890164 +asinpi10103 asinpi -0.6554324978014275 -> -0.22751362426513144 +asinpi10104 asinpi -0.6554324978014276 -> -0.2275136242651315 + +asinpi20001 asinpi inf -> nan invalid +asinpi20002 asinpi -inf -> nan invalid +asinpi20003 asinpi nan -> nan + +------------ +-- atanpi -- +------------ + +atanpi10000 atanpi 0.0 -> 0.0 +atanpi10001 atanpi -0.0 -> -0.0 +atanpi10002 atanpi 3.4028234663852886e+38 -> 0.5 +atanpi10003 atanpi 1.7976931348623157e+308 -> 0.5 +atanpi10004 atanpi -3.4028234663852886e+38 -> -0.5 +atanpi10005 atanpi -1.7976931348623157e+308 -> -0.5 +atanpi10006 atanpi 1.0 -> 0.25 +atanpi10007 atanpi -1.0 -> -0.25 +atanpi10008 atanpi 0.75 -> 0.20483276469913345 +atanpi10009 atanpi 0.03125 -> 0.009943947823589275 +atanpi10010 atanpi 0.0009765625 -> 0.0003108493994100204 +atanpi10011 atanpi 3.0517578125e-05 -> 9.714046816558053e-06 +atanpi10012 atanpi 9.5367431640625e-07 -> 3.0356396311158577e-07 +atanpi10013 atanpi 2.9802322387695312e-08 -> 9.486373847239929e-09 +atanpi10014 atanpi 9.313225746154785e-10 -> 2.964491827262479e-10 +atanpi10015 atanpi 2.9103830456733704e-11 -> 9.264036960195246e-12 +atanpi10016 atanpi 9.094947017729282e-13 -> 2.8950115500610145e-13 +atanpi10017 atanpi 2.842170943040401e-14 -> 9.04691109394067e-15 +atanpi10018 atanpi 8.881784197001252e-16 -> 2.8271597168564594e-16 +atanpi10019 atanpi 2.7755575615628914e-17 -> 8.834874115176436e-18 +atanpi10020 atanpi 8.673617379884035e-19 -> 2.760898160992636e-19 +atanpi10021 atanpi 2.5 -> 0.3788810584091566 +atanpi10022 atanpi 10.0 -> 0.4682744825694464 +atanpi10023 atanpi 1000000.0 -> 0.49999968169011383 +atanpi10024 atanpi 2147483648.0 -> 0.4999999998517754 +atanpi10025 atanpi 7.888609052210118e-31 -> 2.5110222495574236e-31 +atanpi10026 atanpi 1.401298464324817e-45 -> 4.460471546887532e-46 +atanpi10027 atanpi 0.0 -> 0.0 +atanpi10028 atanpi 2.409919865102884e-181 -> 7.671013179729553e-182 +atanpi10029 atanpi 1.401298464324817e-45 -> 4.460471546887532e-46 +atanpi10030 atanpi 0.0 -> 0.0 +atanpi10031 atanpi 5e-324 -> 0.0 +atanpi10032 atanpi -0.23051215708255768 -> -0.07211461946620386 +atanpi10033 atanpi -0.20305541157722473 -> -0.06376756760558985 +atanpi10034 atanpi -2.1964359283447266 -> -0.36400572992968316 +atanpi10035 atanpi -1.2119029760360718 -> -0.2804016904047878 +atanpi10036 atanpi -1.6769415140151978 -> -0.32884096476652747 +atanpi10037 atanpi -3.9201369285583496 -> -0.4204968782420327 +atanpi10038 atanpi 3.327171802520752 -> 0.40706420561165213 +atanpi10039 atanpi -1.3468046188354492 -> -0.2967010012978093 +atanpi10040 atanpi 0.692195475101471 -> 0.1927267050671652 +atanpi10041 atanpi 1.1754943508222875e-38 -> 3.7417147301993126e-39 +atanpi10042 atanpi 2.2250738585072014e-308 -> 7.082630066519554e-309 +atanpi10043 atanpi 2.004168360008973e-292 -> 6.379466025676106e-293 +atanpi10044 atanpi -1.1754943508222875e-38 -> -3.7417147301993126e-39 +atanpi10045 atanpi -2.2250738585072014e-308 -> -7.082630066519554e-309 +atanpi10046 atanpi -2.004168360008973e-292 -> -6.379466025676106e-293 +atanpi10047 atanpi 1.401298464324817e-45 -> 4.460471546887532e-46 +atanpi10048 atanpi 5e-324 -> 0.0 +atanpi10049 atanpi -1.401298464324817e-45 -> -4.460471546887532e-46 +atanpi10050 atanpi -5e-324 -> -0.0 +atanpi10051 atanpi 0.721666157245636 -> 0.19898171607590034 +atanpi10052 atanpi 0.19824542105197906 -> 0.06229575837467328 +atanpi10053 atanpi 0.19824540615081787 -> 0.06229575381085028 +atanpi10054 atanpi 0.198245408653517 -> 0.06229575457735939 +atanpi10055 atanpi 0.049094054847955704 -> 0.01561458616872582 +atanpi10056 atanpi 0.049094051122665405 -> 0.015614584985780253 +atanpi10057 atanpi 0.04909405149251647 -> 0.015614585103224438 +atanpi10058 atanpi 0.049094051492516465 -> 0.015614585103224436 +atanpi10059 atanpi 0.19859454035758972 -> 0.062402677059315244 +atanpi10060 atanpi 0.19859452545642853 -> 0.06240267249610054 +atanpi10061 atanpi 0.1985945401764544 -> 0.06240267700384578 +atanpi10062 atanpi 0.19859454017645436 -> 0.06240267700384577 + +atanpi20001 atanpi inf -> 0.5 +atanpi20002 atanpi -inf -> -0.5 +atanpi20003 atanpi nan -> nan + +------------- +-- atan2pi -- +------------- + +atan2pi10000 atan2pi 0.0 1.0 -> 0.0 +atan2pi10001 atan2pi -0.0 1.0 -> -0.0 +atan2pi10002 atan2pi 0.0 0.0 -> 0.0 +atan2pi10003 atan2pi -0.0 0.0 -> -0.0 +atan2pi10004 atan2pi 0.0 -1.0 -> 1.0 +atan2pi10005 atan2pi -0.0 -1.0 -> -1.0 +atan2pi10006 atan2pi 0.0 -0.0 -> 1.0 +atan2pi10007 atan2pi -0.0 -0.0 -> -1.0 +atan2pi10008 atan2pi 1.0 0.0 -> 0.5 +atan2pi10009 atan2pi 1.0 -0.0 -> 0.5 +atan2pi10010 atan2pi -1.0 0.0 -> -0.5 +atan2pi10011 atan2pi -1.0 -0.0 -> -0.5 +atan2pi10012 atan2pi 3.4028234663852886e+38 3.4028234663852886e+38 -> 0.25 +atan2pi10013 atan2pi 3.4028234663852886e+38 1.7976931348623157e+308 -> 6.025234948519683e-271 +atan2pi10014 atan2pi 1.7976931348623157e+308 3.4028234663852886e+38 -> 0.5 +atan2pi10015 atan2pi 1.7976931348623157e+308 1.7976931348623157e+308 -> 0.25 +atan2pi10016 atan2pi 3.4028234663852886e+38 -3.4028234663852886e+38 -> 0.75 +atan2pi10017 atan2pi 3.4028234663852886e+38 -1.7976931348623157e+308 -> 1.0 +atan2pi10018 atan2pi 1.7976931348623157e+308 -3.4028234663852886e+38 -> 0.5 +atan2pi10019 atan2pi 1.7976931348623157e+308 -1.7976931348623157e+308 -> 0.75 +atan2pi10020 atan2pi -3.4028234663852886e+38 3.4028234663852886e+38 -> -0.25 +atan2pi10021 atan2pi -3.4028234663852886e+38 1.7976931348623157e+308 -> -6.025234948519683e-271 +atan2pi10022 atan2pi -1.7976931348623157e+308 3.4028234663852886e+38 -> -0.5 +atan2pi10023 atan2pi -1.7976931348623157e+308 1.7976931348623157e+308 -> -0.25 +atan2pi10024 atan2pi -3.4028234663852886e+38 -3.4028234663852886e+38 -> -0.75 +atan2pi10025 atan2pi -3.4028234663852886e+38 -1.7976931348623157e+308 -> -1.0 +atan2pi10026 atan2pi -1.7976931348623157e+308 -3.4028234663852886e+38 -> -0.5 +atan2pi10027 atan2pi -1.7976931348623157e+308 -1.7976931348623157e+308 -> -0.75 +atan2pi10028 atan2pi 3.4028234663852886e+38 1.1754943508222875e-38 -> 0.5 +atan2pi10029 atan2pi 3.4028234663852886e+38 2.2250738585072014e-308 -> 0.5 +atan2pi10030 atan2pi 3.4028234663852886e+38 2.004168360008973e-292 -> 0.5 +atan2pi10031 atan2pi 1.7976931348623157e+308 1.1754943508222875e-38 -> 0.5 +atan2pi10032 atan2pi 1.7976931348623157e+308 2.2250738585072014e-308 -> 0.5 +atan2pi10033 atan2pi 1.7976931348623157e+308 2.004168360008973e-292 -> 0.5 +atan2pi10034 atan2pi -3.4028234663852886e+38 -1.1754943508222875e-38 -> -0.5 +atan2pi10035 atan2pi -3.4028234663852886e+38 -2.2250738585072014e-308 -> -0.5 +atan2pi10036 atan2pi -3.4028234663852886e+38 -2.004168360008973e-292 -> -0.5 +atan2pi10037 atan2pi -1.7976931348623157e+308 -1.1754943508222875e-38 -> -0.5 +atan2pi10038 atan2pi -1.7976931348623157e+308 -2.2250738585072014e-308 -> -0.5 +atan2pi10039 atan2pi -1.7976931348623157e+308 -2.004168360008973e-292 -> -0.5 +atan2pi10040 atan2pi -3.4028234663852886e+38 1.1754943508222875e-38 -> -0.5 +atan2pi10041 atan2pi -3.4028234663852886e+38 2.2250738585072014e-308 -> -0.5 +atan2pi10042 atan2pi -3.4028234663852886e+38 2.004168360008973e-292 -> -0.5 +atan2pi10043 atan2pi -1.7976931348623157e+308 1.1754943508222875e-38 -> -0.5 +atan2pi10044 atan2pi -1.7976931348623157e+308 2.2250738585072014e-308 -> -0.5 +atan2pi10045 atan2pi -1.7976931348623157e+308 2.004168360008973e-292 -> -0.5 +atan2pi10046 atan2pi 3.4028234663852886e+38 -1.1754943508222875e-38 -> 0.5 +atan2pi10047 atan2pi 3.4028234663852886e+38 -2.2250738585072014e-308 -> 0.5 +atan2pi10048 atan2pi 3.4028234663852886e+38 -2.004168360008973e-292 -> 0.5 +atan2pi10049 atan2pi 1.7976931348623157e+308 -1.1754943508222875e-38 -> 0.5 +atan2pi10050 atan2pi 1.7976931348623157e+308 -2.2250738585072014e-308 -> 0.5 +atan2pi10051 atan2pi 1.7976931348623157e+308 -2.004168360008973e-292 -> 0.5 +atan2pi10052 atan2pi 3.4028234663852886e+38 1.401298464324817e-45 -> 0.5 +atan2pi10053 atan2pi 3.4028234663852886e+38 5e-324 -> 0.5 +atan2pi10054 atan2pi 1.7976931348623157e+308 1.401298464324817e-45 -> 0.5 +atan2pi10055 atan2pi 1.7976931348623157e+308 5e-324 -> 0.5 +atan2pi10056 atan2pi -3.4028234663852886e+38 -1.401298464324817e-45 -> -0.5 +atan2pi10057 atan2pi -3.4028234663852886e+38 -5e-324 -> -0.5 +atan2pi10058 atan2pi -1.7976931348623157e+308 -1.401298464324817e-45 -> -0.5 +atan2pi10059 atan2pi -1.7976931348623157e+308 -5e-324 -> -0.5 +atan2pi10060 atan2pi -3.4028234663852886e+38 1.401298464324817e-45 -> -0.5 +atan2pi10061 atan2pi -3.4028234663852886e+38 5e-324 -> -0.5 +atan2pi10062 atan2pi -1.7976931348623157e+308 1.401298464324817e-45 -> -0.5 +atan2pi10063 atan2pi -1.7976931348623157e+308 5e-324 -> -0.5 +atan2pi10064 atan2pi 3.4028234663852886e+38 -1.401298464324817e-45 -> 0.5 +atan2pi10065 atan2pi 3.4028234663852886e+38 -5e-324 -> 0.5 +atan2pi10066 atan2pi 1.7976931348623157e+308 -1.401298464324817e-45 -> 0.5 +atan2pi10067 atan2pi 1.7976931348623157e+308 -5e-324 -> 0.5 +atan2pi10068 atan2pi 0.75 1.0 -> 0.20483276469913345 +atan2pi10069 atan2pi -0.75 1.0 -> -0.20483276469913345 +atan2pi10070 atan2pi 0.75 -1.0 -> 0.7951672353008665 +atan2pi10071 atan2pi -0.75 -1.0 -> -0.7951672353008665 +atan2pi10072 atan2pi 0.390625 0.00029000002541579306 -> 0.4997636867632018 +atan2pi10073 atan2pi 0.390625 0.0002899999963119626 -> 0.49976368678691774 +atan2pi10074 atan2pi 0.390625 0.00029 -> 0.4997636867839124 +atan2pi10075 atan2pi 0.390625 0.00028999999999999995 -> 0.4997636867839124 +atan2pi10076 atan2pi 1.390625 0.9296875 -> 0.3124206399141909 +atan2pi10077 atan2pi -0.007568269968032837 -0.0017927357694134116 -> -0.5740351606831543 +atan2pi10078 atan2pi -0.007568269968032837 -0.0017927358858287334 -> -0.574035165319273 +atan2pi10079 atan2pi -0.007568269968032837 -0.001792735857538728 -> -0.5740351641926531 +atan2pi10080 atan2pi -0.007568269968032837 -0.0017927358575387281 -> -0.5740351641926531 +atan2pi10081 atan2pi -0.007568270433694124 -0.0017927357694134116 -> -0.5740351562904277 +atan2pi10082 atan2pi -0.007568270433694124 -0.0017927358858287334 -> -0.574035160926546 +atan2pi10083 atan2pi -0.007568270433694124 -0.001792735857538728 -> -0.5740351597999263 +atan2pi10084 atan2pi -0.007568270433694124 -0.0017927358575387281 -> -0.5740351597999263 +atan2pi10085 atan2pi -0.0075682704267110625 -0.0017927357694134116 -> -0.574035156356301 +atan2pi10086 atan2pi -0.0075682704267110625 -0.0017927358858287334 -> -0.5740351609924195 +atan2pi10087 atan2pi -0.0075682704267110625 -0.001792735857538728 -> -0.5740351598657996 +atan2pi10088 atan2pi -0.0075682704267110625 -0.0017927358575387281 -> -0.5740351598657996 +atan2pi10089 atan2pi -0.007568270426711063 -0.0017927357694134116 -> -0.574035156356301 +atan2pi10090 atan2pi -0.007568270426711063 -0.0017927358858287334 -> -0.5740351609924195 +atan2pi10091 atan2pi -0.007568270426711063 -0.001792735857538728 -> -0.5740351598657996 +atan2pi10092 atan2pi -0.007568270426711063 -0.0017927358575387281 -> -0.5740351598657996 +atan2pi10093 atan2pi 1.0000001192092896 1.0000001192092896 -> 0.25 +atan2pi10094 atan2pi 1.0000001192092896 1.0 -> 0.25000001897274654 +atan2pi10095 atan2pi 1.0000001192092896 1.0000000000000002 -> 0.25000001897274654 +atan2pi10096 atan2pi 1.0 1.0000001192092896 -> 0.24999998102725343 +atan2pi10097 atan2pi 1.0 1.0 -> 0.25 +atan2pi10098 atan2pi 1.0 1.0000000000000002 -> 0.24999999999999997 +atan2pi10099 atan2pi 1.0000000000000002 1.0000001192092896 -> 0.24999998102725346 +atan2pi10100 atan2pi 1.0000000000000002 1.0 -> 0.25000000000000006 +atan2pi10101 atan2pi 1.0000000000000002 1.0000000000000002 -> 0.25 +atan2pi10102 atan2pi 0.2977333664894104 2.9490208625793457 -> 0.032028064084188616 +atan2pi10103 atan2pi -1.346861771417407e-11 1.9269377760089412e-10 -> -0.022212615773618685 +atan2pi10104 atan2pi 0.3338441550731659 2.132420539855957 -> 0.04943220711796703 +atan2pi10105 atan2pi -0.2600718140602112 1.9549157619476318 -> -0.04209909110779262 +atan2pi10106 atan2pi -11223297.0 -0.9756588935852051 -> -0.50000002767118 +atan2pi10107 atan2pi 0.2590596377849579 7.252630710601807 -> 0.011365007240777592 +atan2pi10108 atan2pi 3.7716573755879195e+20 3.019230078687783e+21 -> 0.03955871588967838 +atan2pi10109 atan2pi -0.41808438301086426 3.5622718334198 -> -0.03718816634303022 +atan2pi10110 atan2pi -0.0 3.4028234663852886e+38 -> -0.0 +atan2pi10111 atan2pi -1.401298464324817e-45 3.4028234663852886e+38 -> -1.3108148544731147e-84 +atan2pi10112 atan2pi -8.028930810726831e-218 3.4028234663852886e+38 -> -7.510492618222167e-257 +atan2pi10113 atan2pi 3.943100929260254 -1.3295949697494507 -> 0.6035214600176284 +atan2pi10114 atan2pi -9.648418597407726e-08 1.6104578133839705e-15 -> -0.49999999468695683 +atan2pi10115 atan2pi -1112287276433408.0 4415830896934912.0 -> -0.07854398572055855 +atan2pi10116 atan2pi 9.642213451703619e-17 9.633302400400708e-17 -> 0.2501471543249841 +atan2pi10117 atan2pi 9.642213451703619e-17 9.633301738656218e-17 -> 0.25014716525787717 +atan2pi10118 atan2pi 9.642213451703619e-17 9.633302010283342e-17 -> 0.2501471607702382 +atan2pi10119 atan2pi 9.642212789959129e-17 9.633302400400708e-17 -> 0.25014714340219485 +atan2pi10120 atan2pi 9.642212789959129e-17 9.633301738656218e-17 -> 0.25014715433508794 +atan2pi10121 atan2pi 9.642212789959129e-17 9.633302010283342e-17 -> 0.25014714984744896 +atan2pi10122 atan2pi 9.642213402663379e-17 9.633302400400708e-17 -> 0.25014715351552336 +atan2pi10123 atan2pi 9.642213402663379e-17 9.633301738656218e-17 -> 0.25014716444841645 +atan2pi10124 atan2pi 9.642213402663379e-17 9.633302010283342e-17 -> 0.25014715996077747 +atan2pi10125 atan2pi 1.401298464324817e-45 1.401298464324817e-45 -> 0.25 +atan2pi10126 atan2pi 1.401298464324817e-45 0.0 -> 0.5 +atan2pi10127 atan2pi 1.401298464324817e-45 2.6248243660802892e-105 -> 0.5 +atan2pi10128 atan2pi 0.0 1.401298464324817e-45 -> 0.0 +atan2pi10129 atan2pi 0.0 0.0 -> 0.0 +atan2pi10130 atan2pi 0.0 2.6248243660802892e-105 -> 0.0 +atan2pi10131 atan2pi 2.841564668406762e-105 1.401298464324817e-45 -> 6.454714318268012e-61 +atan2pi10132 atan2pi 2.841564668406762e-105 0.0 -> 0.5 +atan2pi10133 atan2pi 2.841564668406762e-105 2.6248243660802892e-105 -> 0.262614261697575 +atan2pi10134 atan2pi 1.401298464324817e-45 1.401298464324817e-45 -> 0.25 +atan2pi10135 atan2pi 1.401298464324817e-45 0.0 -> 0.5 +atan2pi10136 atan2pi 1.401298464324817e-45 1.6955960445645656e-177 -> 0.5 +atan2pi10137 atan2pi 0.0 1.401298464324817e-45 -> 0.0 +atan2pi10138 atan2pi 0.0 0.0 -> 0.0 +atan2pi10139 atan2pi 0.0 1.6955960445645656e-177 -> 0.0 +atan2pi10140 atan2pi 1.697410900901164e-177 1.401298464324817e-45 -> 3.855728700404359e-133 +atan2pi10141 atan2pi 1.697410900901164e-177 0.0 -> 0.5 +atan2pi10142 atan2pi 1.697410900901164e-177 1.6955960445645656e-177 -> 0.2501702580242645 +atan2pi10143 atan2pi 1.401298464324817e-45 1.401298464324817e-45 -> 0.25 +atan2pi10144 atan2pi 1.401298464324817e-45 0.0 -> 0.5 +atan2pi10145 atan2pi 1.401298464324817e-45 4.642262744535978e-191 -> 0.5 +atan2pi10146 atan2pi 0.0 1.401298464324817e-45 -> 0.0 +atan2pi10147 atan2pi 0.0 0.0 -> 0.0 +atan2pi10148 atan2pi 0.0 4.642262744535978e-191 -> 0.0 +atan2pi10149 atan2pi 4.6493548767531986e-191 1.401298464324817e-45 -> 1.0561173506747795e-146 +atan2pi10150 atan2pi 4.6493548767531986e-191 0.0 -> 0.5 +atan2pi10151 atan2pi 4.6493548767531986e-191 4.642262744535978e-191 -> 0.2502429604243814 +atan2pi10152 atan2pi 1.401298464324817e-45 1.401298464324817e-45 -> 0.25 +atan2pi10153 atan2pi 1.401298464324817e-45 0.0 -> 0.5 +atan2pi10154 atan2pi 1.401298464324817e-45 5e-324 -> 0.5 +atan2pi10155 atan2pi 0.0 1.401298464324817e-45 -> 0.0 +atan2pi10156 atan2pi 0.0 0.0 -> 0.0 +atan2pi10157 atan2pi 0.0 5e-324 -> 0.0 +atan2pi10158 atan2pi 5e-324 1.401298464324817e-45 -> 1.122287531877252e-279 +atan2pi10159 atan2pi 5e-324 0.0 -> 0.5 +atan2pi10160 atan2pi 5e-324 5e-324 -> 0.25 +atan2pi10161 atan2pi 1.1754943508222875e-38 1.1754943508222875e-38 -> 0.25 +atan2pi10162 atan2pi 1.1754943508222875e-38 2.2250738585072014e-308 -> 0.5 +atan2pi10163 atan2pi 1.1754943508222875e-38 2.004168360008973e-292 -> 0.5 +atan2pi10164 atan2pi 2.2250738585072014e-308 1.1754943508222875e-38 -> 6.025235307651693e-271 +atan2pi10165 atan2pi 2.2250738585072014e-308 2.2250738585072014e-308 -> 0.25 +atan2pi10166 atan2pi 2.2250738585072014e-308 2.004168360008973e-292 -> 3.533949646070574e-17 +atan2pi10167 atan2pi 2.004168360008973e-292 1.1754943508222875e-38 -> 5.427049497271944e-255 +atan2pi10168 atan2pi 2.004168360008973e-292 2.2250738585072014e-308 -> 0.49999999999999994 +atan2pi10169 atan2pi 2.004168360008973e-292 2.004168360008973e-292 -> 0.25 +atan2pi10170 atan2pi 1.1754943508222875e-38 -1.1754943508222875e-38 -> 0.75 +atan2pi10171 atan2pi 1.1754943508222875e-38 -2.2250738585072014e-308 -> 0.5 +atan2pi10172 atan2pi 1.1754943508222875e-38 -2.004168360008973e-292 -> 0.5 +atan2pi10173 atan2pi 2.2250738585072014e-308 -1.1754943508222875e-38 -> 1.0 +atan2pi10174 atan2pi 2.2250738585072014e-308 -2.2250738585072014e-308 -> 0.75 +atan2pi10175 atan2pi 2.2250738585072014e-308 -2.004168360008973e-292 -> 1.0 +atan2pi10176 atan2pi 2.004168360008973e-292 -1.1754943508222875e-38 -> 1.0 +atan2pi10177 atan2pi 2.004168360008973e-292 -2.2250738585072014e-308 -> 0.5 +atan2pi10178 atan2pi 2.004168360008973e-292 -2.004168360008973e-292 -> 0.75 +atan2pi10179 atan2pi -1.1754943508222875e-38 1.1754943508222875e-38 -> -0.25 +atan2pi10180 atan2pi -1.1754943508222875e-38 2.2250738585072014e-308 -> -0.5 +atan2pi10181 atan2pi -1.1754943508222875e-38 2.004168360008973e-292 -> -0.5 +atan2pi10182 atan2pi -2.2250738585072014e-308 1.1754943508222875e-38 -> -6.025235307651693e-271 +atan2pi10183 atan2pi -2.2250738585072014e-308 2.2250738585072014e-308 -> -0.25 +atan2pi10184 atan2pi -2.2250738585072014e-308 2.004168360008973e-292 -> -3.533949646070574e-17 +atan2pi10185 atan2pi -2.004168360008973e-292 1.1754943508222875e-38 -> -5.427049497271944e-255 +atan2pi10186 atan2pi -2.004168360008973e-292 2.2250738585072014e-308 -> -0.49999999999999994 +atan2pi10187 atan2pi -2.004168360008973e-292 2.004168360008973e-292 -> -0.25 +atan2pi10188 atan2pi -1.1754943508222875e-38 -1.1754943508222875e-38 -> -0.75 +atan2pi10189 atan2pi -1.1754943508222875e-38 -2.2250738585072014e-308 -> -0.5 +atan2pi10190 atan2pi -1.1754943508222875e-38 -2.004168360008973e-292 -> -0.5 +atan2pi10191 atan2pi -2.2250738585072014e-308 -1.1754943508222875e-38 -> -1.0 +atan2pi10192 atan2pi -2.2250738585072014e-308 -2.2250738585072014e-308 -> -0.75 +atan2pi10193 atan2pi -2.2250738585072014e-308 -2.004168360008973e-292 -> -1.0 +atan2pi10194 atan2pi -2.004168360008973e-292 -1.1754943508222875e-38 -> -1.0 +atan2pi10195 atan2pi -2.004168360008973e-292 -2.2250738585072014e-308 -> -0.5 +atan2pi10196 atan2pi -2.004168360008973e-292 -2.004168360008973e-292 -> -0.75 +atan2pi10197 atan2pi 1.401298464324817e-45 1.401298464324817e-45 -> 0.25 +atan2pi10198 atan2pi 1.401298464324817e-45 5e-324 -> 0.5 +atan2pi10199 atan2pi 5e-324 1.401298464324817e-45 -> 1.122287531877252e-279 +atan2pi10200 atan2pi 5e-324 5e-324 -> 0.25 +atan2pi10201 atan2pi 1.401298464324817e-45 -1.401298464324817e-45 -> 0.75 +atan2pi10202 atan2pi 1.401298464324817e-45 -5e-324 -> 0.5 +atan2pi10203 atan2pi 5e-324 -1.401298464324817e-45 -> 1.0 +atan2pi10204 atan2pi 5e-324 -5e-324 -> 0.75 +atan2pi10205 atan2pi -1.401298464324817e-45 1.401298464324817e-45 -> -0.25 +atan2pi10206 atan2pi -1.401298464324817e-45 5e-324 -> -0.5 +atan2pi10207 atan2pi -5e-324 1.401298464324817e-45 -> -1.122287531877252e-279 +atan2pi10208 atan2pi -5e-324 5e-324 -> -0.25 +atan2pi10209 atan2pi -1.401298464324817e-45 -1.401298464324817e-45 -> -0.75 +atan2pi10210 atan2pi -1.401298464324817e-45 -5e-324 -> -0.5 +atan2pi10211 atan2pi -5e-324 -1.401298464324817e-45 -> -1.0 +atan2pi10212 atan2pi -5e-324 -5e-324 -> -0.75 +atan2pi10213 atan2pi 1.1754943508222875e-38 1.401298464324817e-45 -> 0.4999999620545046 +atan2pi10214 atan2pi 1.1754943508222875e-38 5e-324 -> 0.5 +atan2pi10215 atan2pi 2.2250738585072014e-308 1.401298464324817e-45 -> 5.054333710364945e-264 +atan2pi10216 atan2pi 2.2250738585072014e-308 5e-324 -> 0.49999999999999994 +atan2pi10217 atan2pi 2.004168360008973e-292 1.401298464324817e-45 -> 4.552539082921141e-248 +atan2pi10218 atan2pi 2.004168360008973e-292 5e-324 -> 0.5 +atan2pi10219 atan2pi 1.1754943508222875e-38 -1.401298464324817e-45 -> 0.5000000379454954 +atan2pi10220 atan2pi 1.1754943508222875e-38 -5e-324 -> 0.5 +atan2pi10221 atan2pi 2.2250738585072014e-308 -1.401298464324817e-45 -> 1.0 +atan2pi10222 atan2pi 2.2250738585072014e-308 -5e-324 -> 0.5000000000000001 +atan2pi10223 atan2pi 2.004168360008973e-292 -1.401298464324817e-45 -> 1.0 +atan2pi10224 atan2pi 2.004168360008973e-292 -5e-324 -> 0.5 +atan2pi10225 atan2pi -1.1754943508222875e-38 1.401298464324817e-45 -> -0.4999999620545046 +atan2pi10226 atan2pi -1.1754943508222875e-38 5e-324 -> -0.5 +atan2pi10227 atan2pi -2.2250738585072014e-308 1.401298464324817e-45 -> -5.054333710364945e-264 +atan2pi10228 atan2pi -2.2250738585072014e-308 5e-324 -> -0.49999999999999994 +atan2pi10229 atan2pi -2.004168360008973e-292 1.401298464324817e-45 -> -4.552539082921141e-248 +atan2pi10230 atan2pi -2.004168360008973e-292 5e-324 -> -0.5 +atan2pi10231 atan2pi -1.1754943508222875e-38 -1.401298464324817e-45 -> -0.5000000379454954 +atan2pi10232 atan2pi -1.1754943508222875e-38 -5e-324 -> -0.5 +atan2pi10233 atan2pi -2.2250738585072014e-308 -1.401298464324817e-45 -> -1.0 +atan2pi10234 atan2pi -2.2250738585072014e-308 -5e-324 -> -0.5000000000000001 +atan2pi10235 atan2pi -2.004168360008973e-292 -1.401298464324817e-45 -> -1.0 +atan2pi10236 atan2pi -2.004168360008973e-292 -5e-324 -> -0.5 +atan2pi10237 atan2pi 1.401298464324817e-45 1.1754943508222875e-38 -> 3.7945495388959543e-08 +atan2pi10238 atan2pi 1.401298464324817e-45 2.2250738585072014e-308 -> 0.5 +atan2pi10239 atan2pi 1.401298464324817e-45 2.004168360008973e-292 -> 0.5 +atan2pi10240 atan2pi 5e-324 1.1754943508222875e-38 -> 1.3378709934678696e-286 +atan2pi10241 atan2pi 5e-324 2.2250738585072014e-308 -> 7.067899292141149e-17 +atan2pi10242 atan2pi 5e-324 2.004168360008973e-292 -> 7.846944529866949e-33 +atan2pi10243 atan2pi 1.401298464324817e-45 -1.1754943508222875e-38 -> 0.9999999620545046 +atan2pi10244 atan2pi 1.401298464324817e-45 -2.2250738585072014e-308 -> 0.5 +atan2pi10245 atan2pi 1.401298464324817e-45 -2.004168360008973e-292 -> 0.5 +atan2pi10246 atan2pi 5e-324 -1.1754943508222875e-38 -> 1.0 +atan2pi10247 atan2pi 5e-324 -2.2250738585072014e-308 -> 0.9999999999999999 +atan2pi10248 atan2pi 5e-324 -2.004168360008973e-292 -> 1.0 +atan2pi10249 atan2pi -1.401298464324817e-45 1.1754943508222875e-38 -> -3.7945495388959543e-08 +atan2pi10250 atan2pi -1.401298464324817e-45 2.2250738585072014e-308 -> -0.5 +atan2pi10251 atan2pi -1.401298464324817e-45 2.004168360008973e-292 -> -0.5 +atan2pi10252 atan2pi -5e-324 1.1754943508222875e-38 -> -1.3378709934678696e-286 +atan2pi10253 atan2pi -5e-324 2.2250738585072014e-308 -> -7.067899292141149e-17 +atan2pi10254 atan2pi -5e-324 2.004168360008973e-292 -> -7.846944529866949e-33 +atan2pi10255 atan2pi -1.401298464324817e-45 -1.1754943508222875e-38 -> -0.9999999620545046 +atan2pi10256 atan2pi -1.401298464324817e-45 -2.2250738585072014e-308 -> -0.5 +atan2pi10257 atan2pi -1.401298464324817e-45 -2.004168360008973e-292 -> -0.5 +atan2pi10258 atan2pi -5e-324 -1.1754943508222875e-38 -> -1.0 +atan2pi10259 atan2pi -5e-324 -2.2250738585072014e-308 -> -0.9999999999999999 +atan2pi10260 atan2pi -5e-324 -2.004168360008973e-292 -> -1.0 +atan2pi10261 atan2pi 1.0 -3.4028234663852886e+38 -> 1.0 +atan2pi10262 atan2pi 1.0 -1.7976931348623157e+308 -> 1.0 +atan2pi10263 atan2pi -1.0 -3.4028234663852886e+38 -> -1.0 +atan2pi10264 atan2pi -1.0 -1.7976931348623157e+308 -> -1.0 +atan2pi10265 atan2pi 1.1754943508222875e-38 -3.4028234663852886e+38 -> 1.0 +atan2pi10266 atan2pi 1.1754943508222875e-38 -1.7976931348623157e+308 -> 1.0 +atan2pi10267 atan2pi 2.2250738585072014e-308 -3.4028234663852886e+38 -> 1.0 +atan2pi10268 atan2pi 2.2250738585072014e-308 -1.7976931348623157e+308 -> 1.0 +atan2pi10269 atan2pi 2.004168360008973e-292 -3.4028234663852886e+38 -> 1.0 +atan2pi10270 atan2pi 2.004168360008973e-292 -1.7976931348623157e+308 -> 1.0 +atan2pi10271 atan2pi -1.1754943508222875e-38 -3.4028234663852886e+38 -> -1.0 +atan2pi10272 atan2pi -1.1754943508222875e-38 -1.7976931348623157e+308 -> -1.0 +atan2pi10273 atan2pi -2.2250738585072014e-308 -3.4028234663852886e+38 -> -1.0 +atan2pi10274 atan2pi -2.2250738585072014e-308 -1.7976931348623157e+308 -> -1.0 +atan2pi10275 atan2pi -2.004168360008973e-292 -3.4028234663852886e+38 -> -1.0 +atan2pi10276 atan2pi -2.004168360008973e-292 -1.7976931348623157e+308 -> -1.0 +atan2pi10277 atan2pi 1.401298464324817e-45 -3.4028234663852886e+38 -> 1.0 +atan2pi10278 atan2pi 1.401298464324817e-45 -1.7976931348623157e+308 -> 1.0 +atan2pi10279 atan2pi 5e-324 -3.4028234663852886e+38 -> 1.0 +atan2pi10280 atan2pi 5e-324 -1.7976931348623157e+308 -> 1.0 +atan2pi10281 atan2pi -1.401298464324817e-45 -3.4028234663852886e+38 -> -1.0 +atan2pi10282 atan2pi -1.401298464324817e-45 -1.7976931348623157e+308 -> -1.0 +atan2pi10283 atan2pi -5e-324 -3.4028234663852886e+38 -> -1.0 +atan2pi10284 atan2pi -5e-324 -1.7976931348623157e+308 -> -1.0 +atan2pi10285 atan2pi 1.0 3.4028234663852886e+38 -> 9.354287383057258e-40 +atan2pi10286 atan2pi 1.0 1.7976931348623157e+308 -> 1.770657516629887e-309 +atan2pi10287 atan2pi -1.0 3.4028234663852886e+38 -> -9.354287383057258e-40 +atan2pi10288 atan2pi -1.0 1.7976931348623157e+308 -> -1.770657516629887e-309 +atan2pi10289 atan2pi 1.1754943508222875e-38 3.4028234663852886e+38 -> 1.0995911974752006e-77 +atan2pi10290 atan2pi 1.1754943508222875e-38 1.7976931348623157e+308 -> 0.0 +atan2pi10291 atan2pi 2.2250738585072014e-308 3.4028234663852886e+38 -> 0.0 +atan2pi10292 atan2pi 2.2250738585072014e-308 1.7976931348623157e+308 -> 0.0 +atan2pi10293 atan2pi 2.004168360008973e-292 3.4028234663852886e+38 -> 0.0 +atan2pi10294 atan2pi 2.004168360008973e-292 1.7976931348623157e+308 -> 0.0 +atan2pi10295 atan2pi -1.1754943508222875e-38 3.4028234663852886e+38 -> -1.0995911974752006e-77 +atan2pi10296 atan2pi -1.1754943508222875e-38 1.7976931348623157e+308 -> -0.0 +atan2pi10297 atan2pi -2.2250738585072014e-308 3.4028234663852886e+38 -> -0.0 +atan2pi10298 atan2pi -2.2250738585072014e-308 1.7976931348623157e+308 -> -0.0 +atan2pi10299 atan2pi -2.004168360008973e-292 3.4028234663852886e+38 -> -0.0 +atan2pi10300 atan2pi -2.004168360008973e-292 1.7976931348623157e+308 -> -0.0 +atan2pi10301 atan2pi 1.401298464324817e-45 3.4028234663852886e+38 -> 1.3108148544731147e-84 +atan2pi10302 atan2pi 1.401298464324817e-45 1.7976931348623157e+308 -> 0.0 +atan2pi10303 atan2pi 5e-324 3.4028234663852886e+38 -> 0.0 +atan2pi10304 atan2pi 5e-324 1.7976931348623157e+308 -> 0.0 +atan2pi10305 atan2pi -1.401298464324817e-45 3.4028234663852886e+38 -> -1.3108148544731147e-84 +atan2pi10306 atan2pi -1.401298464324817e-45 1.7976931348623157e+308 -> -0.0 +atan2pi10307 atan2pi -5e-324 3.4028234663852886e+38 -> -0.0 +atan2pi10308 atan2pi -5e-324 1.7976931348623157e+308 -> -0.0 +atan2pi10309 atan2pi 1.1754943508222875e-38 1.0 -> 3.7417147301993126e-39 +atan2pi10310 atan2pi 2.2250738585072014e-308 1.0 -> 7.082630066519554e-309 +atan2pi10311 atan2pi 2.004168360008973e-292 1.0 -> 6.379466025676106e-293 +atan2pi10312 atan2pi -1.1754943508222875e-38 1.0 -> -3.7417147301993126e-39 +atan2pi10313 atan2pi -2.2250738585072014e-308 1.0 -> -7.082630066519554e-309 +atan2pi10314 atan2pi -2.004168360008973e-292 1.0 -> -6.379466025676106e-293 +atan2pi10315 atan2pi 1.401298464324817e-45 1.0 -> 4.460471546887532e-46 +atan2pi10316 atan2pi 5e-324 1.0 -> 0.0 +atan2pi10317 atan2pi -1.401298464324817e-45 1.0 -> -4.460471546887532e-46 +atan2pi10318 atan2pi -5e-324 1.0 -> -0.0 +atan2pi10319 atan2pi 1.1754943508222875e-38 -1.0 -> 1.0 +atan2pi10320 atan2pi 2.2250738585072014e-308 -1.0 -> 1.0 +atan2pi10321 atan2pi 2.004168360008973e-292 -1.0 -> 1.0 +atan2pi10322 atan2pi -1.1754943508222875e-38 -1.0 -> -1.0 +atan2pi10323 atan2pi -2.2250738585072014e-308 -1.0 -> -1.0 +atan2pi10324 atan2pi -2.004168360008973e-292 -1.0 -> -1.0 +atan2pi10325 atan2pi 1.401298464324817e-45 -1.0 -> 1.0 +atan2pi10326 atan2pi 5e-324 -1.0 -> 1.0 +atan2pi10327 atan2pi -1.401298464324817e-45 -1.0 -> -1.0 +atan2pi10328 atan2pi -5e-324 -1.0 -> -1.0 +atan2pi10329 atan2pi -4.3609299609670416e-05 0.05748599022626877 -> -0.0002414722000586593 +atan2pi10330 atan2pi -3.4028234663852886e+38 3.4028234663852886e+38 -> -0.25 +atan2pi10331 atan2pi -3.4028234663852886e+38 2.8061581043948965e+103 -> -3.8599120576714543e-66 +atan2pi10332 atan2pi -9.822050856552661e+114 3.4028234663852886e+38 -> -0.5 +atan2pi10333 atan2pi -9.822050856552661e+114 2.8061581043948965e+103 -> -0.4999999999990906 +atan2pi10334 atan2pi 3.4028234663852886e+38 3.4028234663852886e+38 -> 0.25 +atan2pi10335 atan2pi 3.4028234663852886e+38 1.7976931348623157e+308 -> 6.025234948519683e-271 +atan2pi10336 atan2pi 2.962077767429005e+64 3.4028234663852886e+38 -> 0.5 +atan2pi10337 atan2pi 2.962077767429005e+64 1.7976931348623157e+308 -> 5.244825263740446e-245 +atan2pi10338 atan2pi 2.9620777674290047e+64 3.4028234663852886e+38 -> 0.5 +atan2pi10339 atan2pi 2.9620777674290047e+64 1.7976931348623157e+308 -> 5.244825263740445e-245 +atan2pi10340 atan2pi -3.4028234663852886e+38 3.4028234663852886e+38 -> -0.25 +atan2pi10341 atan2pi -3.4028234663852886e+38 1.7976931348623157e+308 -> -6.025234948519683e-271 +atan2pi10342 atan2pi -1.7976931348623157e+308 3.4028234663852886e+38 -> -0.5 +atan2pi10343 atan2pi -1.7976931348623157e+308 1.7976931348623157e+308 -> -0.25 + +atan2pi20000 atan2pi inf 0 -> 0.5 +atan2pi20001 atan2pi -inf 0 -> -0.5 +atan2pi20002 atan2pi nan 0 -> nan + +atan2pi20000 atan2pi inf -0 -> 0.5 +atan2pi20001 atan2pi -inf -0 -> -0.5 +atan2pi20002 atan2pi nan -0 -> nan + +atan2pi20003 atan2pi inf 1 -> 0.5 +atan2pi20004 atan2pi -inf 1 -> -0.5 +atan2pi20005 atan2pi nan 1 -> nan + +atan2pi20006 atan2pi inf -1 -> 0.5 +atan2pi20007 atan2pi -inf -1 -> -0.5 +atan2pi20008 atan2pi nan -1 -> nan + +atan2pi20009 atan2pi inf inf -> 0.25 +atan2pi20010 atan2pi -inf inf -> -0.25 +atan2pi20011 atan2pi nan inf -> nan + +atan2pi20012 atan2pi inf -inf -> .75 +atan2pi20013 atan2pi -inf -inf -> -.75 +atan2pi20014 atan2pi nan -inf -> nan + +atan2pi20015 atan2pi inf nan -> nan +atan2pi20016 atan2pi -inf nan -> nan +atan2pi20017 atan2pi nan nan -> nan + +atan2pi30000 atan2pi 0 inf -> 0.0 +atan2pi30001 atan2pi 0 -inf -> 1.0 +atan2pi30002 atan2pi 0 nan -> nan + +atan2pi30000 atan2pi -0 inf -> -0.0 +atan2pi30001 atan2pi -0 -inf -> -1.0 +atan2pi30002 atan2pi -0 nan -> nan + +atan2pi30003 atan2pi 1 inf -> 0.0 +atan2pi30004 atan2pi 1 -inf -> 1.0 +atan2pi30005 atan2pi 1 nan -> nan + +atan2pi30006 atan2pi -1 inf -> -0.0 +atan2pi30007 atan2pi -1 -inf -> -1.0 +atan2pi30008 atan2pi -1 nan -> nan + +----------- +-- cospi -- +----------- + +cospi10000 cospi 0.0 -> 1.0 +cospi10001 cospi -0.0 -> 1.0 +cospi10002 cospi 1.1754943508222875e-38 -> 1.0 +cospi10003 cospi 2.2250738585072014e-308 -> 1.0 +cospi10004 cospi 2.004168360008973e-292 -> 1.0 +cospi10005 cospi -1.1754943508222875e-38 -> 1.0 +cospi10006 cospi -2.2250738585072014e-308 -> 1.0 +cospi10007 cospi -2.004168360008973e-292 -> 1.0 +cospi10008 cospi 1.401298464324817e-45 -> 1.0 +cospi10009 cospi 5e-324 -> 1.0 +cospi10010 cospi -1.401298464324817e-45 -> 1.0 +cospi10011 cospi -5e-324 -> 1.0 +cospi10012 cospi 3.4028234663852886e+38 -> 1.0 +cospi10013 cospi 1.7976931348623157e+308 -> 1.0 +cospi10014 cospi -3.4028234663852886e+38 -> 1.0 +cospi10015 cospi -1.7976931348623157e+308 -> 1.0 +cospi10016 cospi 1.0 -> -1.0 +cospi10017 cospi -1.0 -> -1.0 +cospi10018 cospi 2.0 -> 1.0 +cospi10019 cospi -2.0 -> 1.0 +cospi10020 cospi 3.0 -> -1.0 +cospi10021 cospi -3.0 -> -1.0 +cospi10022 cospi 0.10000000149011612 -> 0.9510565148485406 +cospi10023 cospi 0.09999999403953552 -> 0.9510565220816052 +cospi10024 cospi 0.1 -> 0.9510565162951535 +cospi10025 cospi 0.09999999999999999 -> 0.9510565162951535 +cospi10026 cospi -0.09999999403953552 -> 0.9510565220816052 +cospi10027 cospi -0.10000000149011612 -> 0.9510565148485406 +cospi10028 cospi -0.09999999999999999 -> 0.9510565162951535 +cospi10029 cospi -0.1 -> 0.9510565162951535 +cospi10030 cospi 0.20000000298023224 -> 0.8090169888717047 +cospi10031 cospi 0.19999998807907104 -> 0.8090170163879177 +cospi10032 cospi 0.2 -> 0.8090169943749475 +cospi10033 cospi 0.19999999999999998 -> 0.8090169943749475 +cospi10034 cospi -0.19999998807907104 -> 0.8090170163879177 +cospi10035 cospi -0.20000000298023224 -> 0.8090169888717047 +cospi10036 cospi -0.19999999999999998 -> 0.8090169943749475 +cospi10037 cospi -0.2 -> 0.8090169943749475 +cospi10038 cospi 0.30000001192092896 -> 0.5877852219942177 +cospi10039 cospi 0.29999998211860657 -> 0.5877852977398548 +cospi10040 cospi 0.30000000000000004 -> 0.587785252292473 +cospi10041 cospi 0.3 -> 0.5877852522924731 +cospi10042 cospi -0.29999998211860657 -> 0.5877852977398548 +cospi10043 cospi -0.30000001192092896 -> 0.5877852219942177 +cospi10044 cospi -0.3 -> 0.5877852522924731 +cospi10045 cospi -0.30000000000000004 -> 0.587785252292473 +cospi10046 cospi 0.4000000059604645 -> 0.3090169765660799 +cospi10047 cospi 0.3999999761581421 -> 0.3090170656104165 +cospi10048 cospi 0.4 -> 0.30901699437494734 +cospi10049 cospi 0.39999999999999997 -> 0.3090169943749475 +cospi10050 cospi -0.3999999761581421 -> 0.3090170656104165 +cospi10051 cospi -0.4000000059604645 -> 0.3090169765660799 +cospi10052 cospi -0.39999999999999997 -> 0.3090169943749475 +cospi10053 cospi -0.4 -> 0.30901699437494734 +cospi10054 cospi 0.5 -> 0.0 +cospi10055 cospi -0.5 -> 0.0 +cospi10056 cospi 0.6000000238418579 -> -0.3090170656104165 +cospi10057 cospi 0.5999999642372131 -> -0.30901688752174056 +cospi10058 cospi 0.6000000000000001 -> -0.3090169943749477 +cospi10059 cospi 0.6 -> -0.30901699437494734 +cospi10060 cospi -0.5999999642372131 -> -0.30901688752174056 +cospi10061 cospi -0.6000000238418579 -> -0.3090170656104165 +cospi10062 cospi -0.6 -> -0.30901699437494734 +cospi10063 cospi -0.6000000000000001 -> -0.3090169943749477 +cospi10064 cospi 0.7000000476837158 -> -0.5877853734854867 +cospi10065 cospi 0.699999988079071 -> -0.5877852219942177 +cospi10066 cospi 0.7000000000000001 -> -0.5877852522924732 +cospi10067 cospi 0.7 -> -0.587785252292473 +cospi10068 cospi -0.699999988079071 -> -0.5877852219942177 +cospi10069 cospi -0.7000000476837158 -> -0.5877853734854867 +cospi10070 cospi -0.7 -> -0.587785252292473 +cospi10071 cospi -0.7000000000000001 -> -0.5877852522924732 +cospi10072 cospi 0.800000011920929 -> -0.8090170163879177 +cospi10073 cospi 0.7999999523162842 -> -0.8090169063230551 +cospi10074 cospi 0.8 -> -0.8090169943749475 +cospi10075 cospi 0.7999999999999999 -> -0.8090169943749473 +cospi10076 cospi -0.7999999523162842 -> -0.8090169063230551 +cospi10077 cospi -0.800000011920929 -> -0.8090170163879177 +cospi10078 cospi -0.7999999999999999 -> -0.8090169943749473 +cospi10079 cospi -0.8 -> -0.8090169943749475 +cospi10080 cospi 0.9000000357627869 -> -0.9510565510138584 +cospi10081 cospi 0.8999999761581421 -> -0.9510564931493437 +cospi10082 cospi 0.9 -> -0.9510565162951536 +cospi10083 cospi 0.8999999999999999 -> -0.9510565162951535 +cospi10084 cospi -0.8999999761581421 -> -0.9510564931493437 +cospi10085 cospi -0.9000000357627869 -> -0.9510565510138584 +cospi10086 cospi -0.8999999999999999 -> -0.9510565162951535 +cospi10087 cospi -0.9 -> -0.9510565162951536 +cospi10088 cospi 1.100000023841858 -> -0.9510564931493437 +cospi10089 cospi 1.0999999046325684 -> -0.9510566088783399 +cospi10090 cospi 1.1 -> -0.9510565162951535 +cospi10091 cospi 1.0999999999999999 -> -0.9510565162951538 +cospi10092 cospi -1.0999999046325684 -> -0.9510566088783399 +cospi10093 cospi -1.100000023841858 -> -0.9510564931493437 +cospi10094 cospi -1.0999999999999999 -> -0.9510565162951538 +cospi10095 cospi -1.1 -> -0.9510565162951535 +cospi10096 cospi 1.2000000476837158 -> -0.8090169063230551 +cospi10097 cospi 1.1999999284744263 -> -0.8090171264527518 +cospi10098 cospi 1.2000000000000002 -> -0.8090169943749471 +cospi10099 cospi 1.2 -> -0.8090169943749475 +cospi10100 cospi -1.1999999284744263 -> -0.8090171264527518 +cospi10101 cospi -1.2000000476837158 -> -0.8090169063230551 +cospi10102 cospi -1.2 -> -0.8090169943749475 +cospi10103 cospi -1.2000000000000002 -> -0.8090169943749471 +cospi10104 cospi 1.3000000715255737 -> -0.587785070502928 +cospi10105 cospi 1.2999999523162842 -> -0.5877853734854867 +cospi10106 cospi 1.3 -> -0.587785252292473 +cospi10107 cospi 1.2999999999999998 -> -0.5877852522924736 +cospi10108 cospi -1.2999999523162842 -> -0.5877853734854867 +cospi10109 cospi -1.3000000715255737 -> -0.587785070502928 +cospi10110 cospi -1.2999999999999998 -> -0.5877852522924736 +cospi10111 cospi -1.3 -> -0.587785252292473 +cospi10112 cospi 1.4000000953674316 -> -0.3090167094330538 +cospi10113 cospi 1.399999976158142 -> -0.3090170656104165 +cospi10114 cospi 1.4000000000000001 -> -0.309016994374947 +cospi10115 cospi 1.4 -> -0.3090169943749477 +cospi10116 cospi -1.399999976158142 -> -0.3090170656104165 +cospi10117 cospi -1.4000000953674316 -> -0.3090167094330538 +cospi10118 cospi -1.4 -> -0.3090169943749477 +cospi10119 cospi -1.4000000000000001 -> -0.309016994374947 +cospi10120 cospi 1.5 -> 0.0 +cospi10121 cospi -1.5 -> 0.0 +cospi10122 cospi 2.5 -> 0.0 +cospi10123 cospi -2.5 -> 0.0 +cospi10124 cospi 3.5 -> 0.0 +cospi10125 cospi -3.5 -> 0.0 +cospi10126 cospi 4.5 -> 0.0 +cospi10127 cospi -4.5 -> 0.0 +cospi10128 cospi 0.24000000953674316 -> 0.728968606911995 +cospi10129 cospi 0.23999999463558197 -> 0.728968638957958 +cospi10130 cospi 0.24000000000000002 -> 0.7289686274214114 +cospi10131 cospi 0.24 -> 0.7289686274214116 +cospi10132 cospi -0.23999999463558197 -> 0.728968638957958 +cospi10133 cospi -0.24000000953674316 -> 0.728968606911995 +cospi10134 cospi -0.24 -> 0.7289686274214116 +cospi10135 cospi -0.24000000000000002 -> 0.7289686274214114 +cospi10136 cospi 0.49000000953674316 -> 0.03141072913234975 +cospi10137 cospi 0.4899999797344208 -> 0.03141082271290761 +cospi10138 cospi 0.49000000000000005 -> 0.031410759078128146 +cospi10139 cospi 0.49 -> 0.03141075907812832 +cospi10140 cospi -0.4899999797344208 -> 0.03141082271290761 +cospi10141 cospi -0.49000000953674316 -> 0.03141072913234975 +cospi10142 cospi -0.49 -> 0.03141075907812832 +cospi10143 cospi -0.49000000000000005 -> 0.031410759078128146 +cospi10144 cospi 0.5100000500679016 -> -0.031410916293465214 +cospi10145 cospi 0.5099999904632568 -> -0.03141072913234975 +cospi10146 cospi 0.51 -> -0.03141075907812832 +cospi10147 cospi 0.5099999999999999 -> -0.03141075907812797 +cospi10148 cospi -0.5099999904632568 -> -0.03141072913234975 +cospi10149 cospi -0.5100000500679016 -> -0.031410916293465214 +cospi10150 cospi -0.5099999999999999 -> -0.03141075907812797 +cospi10151 cospi -0.51 -> -0.03141075907812832 +cospi10152 cospi 0.7600000500679016 -> -0.7289687350958375 +cospi10153 cospi 0.7599999904632568 -> -0.728968606911995 +cospi10154 cospi 0.76 -> -0.7289686274214116 +cospi10155 cospi 0.7599999999999999 -> -0.7289686274214113 +cospi10156 cospi -0.7599999904632568 -> -0.728968606911995 +cospi10157 cospi -0.7600000500679016 -> -0.7289687350958375 +cospi10158 cospi -0.7599999999999999 -> -0.7289686274214113 +cospi10159 cospi -0.76 -> -0.7289686274214116 +cospi10160 cospi 1.2400000095367432 -> -0.728968606911995 +cospi10161 cospi 1.2399998903274536 -> -0.7289688632796544 +cospi10162 cospi 1.2400000000000002 -> -0.7289686274214111 +cospi10163 cospi 1.24 -> -0.7289686274214116 +cospi10164 cospi -1.2399998903274536 -> -0.7289688632796544 +cospi10165 cospi -1.2400000095367432 -> -0.728968606911995 +cospi10166 cospi -1.24 -> -0.7289686274214116 +cospi10167 cospi -1.2400000000000002 -> -0.7289686274214111 +cospi10168 cospi 1.4900000095367432 -> -0.03141072913234975 +cospi10169 cospi 1.4899998903274536 -> -0.03141110345457958 +cospi10170 cospi 1.4900000000000002 -> -0.031410759078127626 +cospi10171 cospi 1.49 -> -0.03141075907812832 +cospi10172 cospi -1.4899998903274536 -> -0.03141110345457958 +cospi10173 cospi -1.4900000095367432 -> -0.03141072913234975 +cospi10174 cospi -1.49 -> -0.03141075907812832 +cospi10175 cospi -1.4900000000000002 -> -0.031410759078127626 +cospi10176 cospi 1.5100001096725464 -> 0.03141110345457958 +cospi10177 cospi 1.5099999904632568 -> 0.03141072913234975 +cospi10178 cospi 1.51 -> 0.03141075907812832 +cospi10179 cospi 1.5099999999999998 -> 0.031410759078127626 +cospi10180 cospi -1.5099999904632568 -> 0.03141072913234975 +cospi10181 cospi -1.5100001096725464 -> 0.03141110345457958 +cospi10182 cospi -1.5099999999999998 -> 0.031410759078127626 +cospi10183 cospi -1.51 -> 0.03141075907812832 +cospi10184 cospi 1.7600001096725464 -> 0.7289688632796544 +cospi10185 cospi 1.7599999904632568 -> 0.728968606911995 +cospi10186 cospi 1.76 -> 0.7289686274214116 +cospi10187 cospi 1.7599999999999998 -> 0.7289686274214111 +cospi10188 cospi -1.7599999904632568 -> 0.728968606911995 +cospi10189 cospi -1.7600001096725464 -> 0.7289688632796544 +cospi10190 cospi -1.7599999999999998 -> 0.7289686274214111 +cospi10191 cospi -1.76 -> 0.7289686274214116 +cospi10192 cospi 100.24000549316406 -> 0.7289568138891386 +cospi10193 cospi 100.23999786376953 -> 0.7289732215142216 +cospi10194 cospi 100.24000000000001 -> 0.728968627421392 +cospi10195 cospi 100.24 -> 0.7289686274214225 +cospi10196 cospi -100.23999786376953 -> 0.7289732215142216 +cospi10197 cospi -100.24000549316406 -> 0.7289568138891386 +cospi10198 cospi -100.24 -> 0.7289686274214225 +cospi10199 cospi -100.24000000000001 -> 0.728968627421392 +cospi10200 cospi 100.49000549316406 -> 0.031393510305016 +cospi10201 cospi 100.48999786376953 -> 0.031417466931812604 +cospi10202 cospi 100.49000000000001 -> 0.03141075907809974 +cospi10203 cospi 100.49 -> 0.031410759078144355 +cospi10204 cospi -100.48999786376953 -> 0.031417466931812604 +cospi10205 cospi -100.49000549316406 -> 0.031393510305016 +cospi10206 cospi -100.49 -> 0.031410759078144355 +cospi10207 cospi -100.49000000000001 -> 0.03141075907809974 +cospi10208 cospi 100.51000213623047 -> -0.031417466931812604 +cospi10209 cospi 100.50999450683594 -> -0.031393510305016 +cospi10210 cospi 100.51 -> -0.031410759078144355 +cospi10211 cospi 100.50999999999999 -> -0.03141075907809974 +cospi10212 cospi -100.50999450683594 -> -0.031393510305016 +cospi10213 cospi -100.51000213623047 -> -0.031417466931812604 +cospi10214 cospi -100.50999999999999 -> -0.03141075907809974 +cospi10215 cospi -100.51 -> -0.031410759078144355 +cospi10216 cospi 100.76000213623047 -> -0.7289732215142216 +cospi10217 cospi 100.75999450683594 -> -0.7289568138891386 +cospi10218 cospi 100.76 -> -0.7289686274214225 +cospi10219 cospi 100.75999999999999 -> -0.728968627421392 +cospi10220 cospi -100.75999450683594 -> -0.7289568138891386 +cospi10221 cospi -100.76000213623047 -> -0.7289732215142216 +cospi10222 cospi -100.75999999999999 -> -0.728968627421392 +cospi10223 cospi -100.76 -> -0.7289686274214225 +cospi10224 cospi 1234.56005859375 -> -0.1875621285825296 +cospi10225 cospi 1234.5599365234375 -> -0.18718542559099033 +cospi10226 cospi 1234.5600000000002 -> -0.1873813145862579 +cospi10227 cospi 1234.56 -> -0.18738131458555624 +cospi10228 cospi -1234.5599365234375 -> -0.18718542559099033 +cospi10229 cospi -1234.56005859375 -> -0.1875621285825296 +cospi10230 cospi -1234.56 -> -0.18738131458555624 +cospi10231 cospi -1234.5600000000002 -> -0.1873813145862579 +cospi10232 cospi 2097151.375 -> -0.3826834323650898 +cospi10233 cospi -2097151.375 -> -0.3826834323650898 +cospi10234 cospi 2097151.625 -> 0.3826834323650898 +cospi10235 cospi -2097151.625 -> 0.3826834323650898 +cospi10236 cospi 1125899906842624.0 -> 1.0 +cospi10237 cospi 1125899839733760.0 -> 1.0 +cospi10238 cospi 1125899906842623.4 -> -0.3826834323650898 +cospi10239 cospi -1125899839733760.0 -> 1.0 +cospi10240 cospi -1125899906842624.0 -> 1.0 +cospi10241 cospi -1125899906842623.4 -> -0.3826834323650898 +cospi10242 cospi 1125899906842624.0 -> 1.0 +cospi10243 cospi 1125899839733760.0 -> 1.0 +cospi10244 cospi 1125899906842623.6 -> 0.3826834323650898 +cospi10245 cospi -1125899839733760.0 -> 1.0 +cospi10246 cospi -1125899906842624.0 -> 1.0 +cospi10247 cospi -1125899906842623.6 -> 0.3826834323650898 +cospi10248 cospi 2.305843009213694e+18 -> 1.0 +cospi10249 cospi 2.3058428717747405e+18 -> 1.0 +cospi10250 cospi 2.3058430092136937e+18 -> 1.0 +cospi10251 cospi -2.3058428717747405e+18 -> 1.0 +cospi10252 cospi -2.305843009213694e+18 -> 1.0 +cospi10253 cospi -2.3058430092136937e+18 -> 1.0 +cospi10254 cospi 2.305843009213694e+18 -> 1.0 +cospi10255 cospi 2.3058428717747405e+18 -> 1.0 +cospi10256 cospi 2.3058430092136937e+18 -> 1.0 +cospi10257 cospi -2.3058428717747405e+18 -> 1.0 +cospi10258 cospi -2.305843009213694e+18 -> 1.0 +cospi10259 cospi -2.3058430092136937e+18 -> 1.0 +cospi10260 cospi 1.298074214633707e+33 -> 1.0 +cospi10261 cospi 1.2980741372624545e+33 -> 1.0 +cospi10262 cospi 1.2980742146337068e+33 -> 1.0 +cospi10263 cospi -1.2980741372624545e+33 -> 1.0 +cospi10264 cospi -1.298074214633707e+33 -> 1.0 +cospi10265 cospi -1.2980742146337068e+33 -> 1.0 +cospi10266 cospi 1.298074214633707e+33 -> 1.0 +cospi10267 cospi 1.2980741372624545e+33 -> 1.0 +cospi10268 cospi 1.2980742146337068e+33 -> 1.0 +cospi10269 cospi -1.2980741372624545e+33 -> 1.0 +cospi10270 cospi -1.298074214633707e+33 -> 1.0 +cospi10271 cospi -1.2980742146337068e+33 -> 1.0 +cospi10272 cospi 0.419987291097641 -> 0.24872855880609973 +cospi10273 cospi -70.50994110107422 -> -0.031225813413226215 +cospi10274 cospi -70.50994873046875 -> -0.031249770165979865 +cospi10275 cospi -70.50994733870746 -> -0.03124539995406979 +cospi10276 cospi 0.4801006019115448 -> 0.06247508982524335 +cospi10277 cospi 0.4801005721092224 -> 0.06247518326910248 +cospi10278 cospi 0.48010059600345173 -> 0.062475108349807 +cospi10279 cospi 0.4801005960034517 -> 0.06247510834980717 +cospi10280 cospi -629430784.0 -> 1.0 +cospi10281 cospi -629430848.0 -> 1.0 +cospi10282 cospi -629430785.4800935 -> -0.06249742289199851 +cospi10283 cospi -629430785.4800936 -> -0.06249704911708015 + +cospi20001 cospi inf -> nan invalid +cospi20002 cospi -inf -> nan invalid +cospi20003 cospi nan -> nan + +----------- +-- sinpi -- +----------- + +sinpi10000 sinpi 0.0 -> 0.0 +sinpi10001 sinpi -0.0 -> -0.0 +sinpi10002 sinpi 1.1754943508222875e-38 -> 3.6929244168796014e-38 +sinpi10003 sinpi 2.2250738585072014e-308 -> 6.990275687580919e-308 +sinpi10004 sinpi 2.004168360008973e-292 -> 6.296280596361293e-292 +sinpi10005 sinpi -1.1754943508222875e-38 -> -3.6929244168796014e-38 +sinpi10006 sinpi -2.2250738585072014e-308 -> -6.990275687580919e-308 +sinpi10007 sinpi -2.004168360008973e-292 -> -6.296280596361293e-292 +sinpi10008 sinpi 1.401298464324817e-45 -> 4.402308961009504e-45 +sinpi10009 sinpi 5e-324 -> 1.5e-323 +sinpi10010 sinpi -1.401298464324817e-45 -> -4.402308961009504e-45 +sinpi10011 sinpi -5e-324 -> -1.5e-323 +sinpi10012 sinpi 3.4028234663852886e+38 -> 0.0 +sinpi10013 sinpi 1.7976931348623157e+308 -> 0.0 +sinpi10014 sinpi -3.4028234663852886e+38 -> -0.0 +sinpi10015 sinpi -1.7976931348623157e+308 -> -0.0 +sinpi10016 sinpi 1.0 -> 0.0 +sinpi10017 sinpi -1.0 -> -0.0 +sinpi10018 sinpi 2.0 -> 0.0 +sinpi10019 sinpi -2.0 -> -0.0 +sinpi10020 sinpi 3.0 -> 0.0 +sinpi10021 sinpi -3.0 -> -0.0 +sinpi10022 sinpi 0.10000000149011612 -> 0.30901699882716427 +sinpi10023 sinpi 0.09999999403953552 -> 0.3090169765660799 +sinpi10024 sinpi 0.1 -> 0.30901699437494745 +sinpi10025 sinpi 0.09999999999999999 -> 0.3090169943749474 +sinpi10026 sinpi -0.09999999403953552 -> -0.3090169765660799 +sinpi10027 sinpi -0.10000000149011612 -> -0.30901699882716427 +sinpi10028 sinpi -0.09999999999999999 -> -0.3090169943749474 +sinpi10029 sinpi -0.1 -> -0.30901699437494745 +sinpi10030 sinpi 0.20000000298023224 -> 0.5877852598670369 +sinpi10031 sinpi 0.19999998807907104 -> 0.5877852219942177 +sinpi10032 sinpi 0.2 -> 0.5877852522924731 +sinpi10033 sinpi 0.19999999999999998 -> 0.5877852522924731 +sinpi10034 sinpi -0.19999998807907104 -> -0.5877852219942177 +sinpi10035 sinpi -0.20000000298023224 -> -0.5877852598670369 +sinpi10036 sinpi -0.19999999999999998 -> -0.5877852522924731 +sinpi10037 sinpi -0.2 -> -0.5877852522924731 +sinpi10038 sinpi 0.30000001192092896 -> 0.8090170163879177 +sinpi10039 sinpi 0.29999998211860657 -> 0.80901696135549 +sinpi10040 sinpi 0.30000000000000004 -> 0.8090169943749475 +sinpi10041 sinpi 0.3 -> 0.8090169943749475 +sinpi10042 sinpi -0.29999998211860657 -> -0.80901696135549 +sinpi10043 sinpi -0.30000001192092896 -> -0.8090170163879177 +sinpi10044 sinpi -0.3 -> -0.8090169943749475 +sinpi10045 sinpi -0.30000000000000004 -> -0.8090169943749475 +sinpi10046 sinpi 0.4000000059604645 -> 0.9510565220816052 +sinpi10047 sinpi 0.3999999761581421 -> 0.9510564931493437 +sinpi10048 sinpi 0.4 -> 0.9510565162951536 +sinpi10049 sinpi 0.39999999999999997 -> 0.9510565162951535 +sinpi10050 sinpi -0.3999999761581421 -> -0.9510564931493437 +sinpi10051 sinpi -0.4000000059604645 -> -0.9510565220816052 +sinpi10052 sinpi -0.39999999999999997 -> -0.9510565162951535 +sinpi10053 sinpi -0.4 -> -0.9510565162951536 +sinpi10054 sinpi 0.5 -> 1.0 +sinpi10055 sinpi -0.5 -> -1.0 +sinpi10056 sinpi 0.6000000238418579 -> 0.9510564931493437 +sinpi10057 sinpi 0.5999999642372131 -> 0.9510565510138584 +sinpi10058 sinpi 0.6000000000000001 -> 0.9510565162951535 +sinpi10059 sinpi 0.6 -> 0.9510565162951536 +sinpi10060 sinpi -0.5999999642372131 -> -0.9510565510138584 +sinpi10061 sinpi -0.6000000238418579 -> -0.9510564931493437 +sinpi10062 sinpi -0.6 -> -0.9510565162951536 +sinpi10063 sinpi -0.6000000000000001 -> -0.9510565162951535 +sinpi10064 sinpi 0.7000000476837158 -> 0.8090169063230551 +sinpi10065 sinpi 0.699999988079071 -> 0.8090170163879177 +sinpi10066 sinpi 0.7000000000000001 -> 0.8090169943749473 +sinpi10067 sinpi 0.7 -> 0.8090169943749475 +sinpi10068 sinpi -0.699999988079071 -> -0.8090170163879177 +sinpi10069 sinpi -0.7000000476837158 -> -0.8090169063230551 +sinpi10070 sinpi -0.7 -> -0.8090169943749475 +sinpi10071 sinpi -0.7000000000000001 -> -0.8090169943749473 +sinpi10072 sinpi 0.800000011920929 -> 0.5877852219942177 +sinpi10073 sinpi 0.7999999523162842 -> 0.5877853734854867 +sinpi10074 sinpi 0.8 -> 0.587785252292473 +sinpi10075 sinpi 0.7999999999999999 -> 0.5877852522924732 +sinpi10076 sinpi -0.7999999523162842 -> -0.5877853734854867 +sinpi10077 sinpi -0.800000011920929 -> -0.5877852219942177 +sinpi10078 sinpi -0.7999999999999999 -> -0.5877852522924732 +sinpi10079 sinpi -0.8 -> -0.587785252292473 +sinpi10080 sinpi 0.9000000357627869 -> 0.30901688752174056 +sinpi10081 sinpi 0.8999999761581421 -> 0.3090170656104165 +sinpi10082 sinpi 0.9 -> 0.30901699437494734 +sinpi10083 sinpi 0.8999999999999999 -> 0.3090169943749477 +sinpi10084 sinpi -0.8999999761581421 -> -0.3090170656104165 +sinpi10085 sinpi -0.9000000357627869 -> -0.30901688752174056 +sinpi10086 sinpi -0.8999999999999999 -> -0.3090169943749477 +sinpi10087 sinpi -0.9 -> -0.30901699437494734 +sinpi10088 sinpi 1.100000023841858 -> -0.3090170656104165 +sinpi10089 sinpi 1.0999999046325684 -> -0.3090167094330538 +sinpi10090 sinpi 1.1 -> -0.3090169943749477 +sinpi10091 sinpi 1.0999999999999999 -> -0.309016994374947 +sinpi10092 sinpi -1.0999999046325684 -> 0.3090167094330538 +sinpi10093 sinpi -1.100000023841858 -> 0.3090170656104165 +sinpi10094 sinpi -1.0999999999999999 -> 0.309016994374947 +sinpi10095 sinpi -1.1 -> 0.3090169943749477 +sinpi10096 sinpi 1.2000000476837158 -> -0.5877853734854867 +sinpi10097 sinpi 1.1999999284744263 -> -0.587785070502928 +sinpi10098 sinpi 1.2000000000000002 -> -0.5877852522924736 +sinpi10099 sinpi 1.2 -> -0.587785252292473 +sinpi10100 sinpi -1.1999999284744263 -> 0.587785070502928 +sinpi10101 sinpi -1.2000000476837158 -> 0.5877853734854867 +sinpi10102 sinpi -1.2 -> 0.587785252292473 +sinpi10103 sinpi -1.2000000000000002 -> 0.5877852522924736 +sinpi10104 sinpi 1.3000000715255737 -> -0.8090171264527518 +sinpi10105 sinpi 1.2999999523162842 -> -0.8090169063230551 +sinpi10106 sinpi 1.3 -> -0.8090169943749475 +sinpi10107 sinpi 1.2999999999999998 -> -0.8090169943749471 +sinpi10108 sinpi -1.2999999523162842 -> 0.8090169063230551 +sinpi10109 sinpi -1.3000000715255737 -> 0.8090171264527518 +sinpi10110 sinpi -1.2999999999999998 -> 0.8090169943749471 +sinpi10111 sinpi -1.3 -> 0.8090169943749475 +sinpi10112 sinpi 1.4000000953674316 -> -0.9510566088783399 +sinpi10113 sinpi 1.399999976158142 -> -0.9510564931493437 +sinpi10114 sinpi 1.4000000000000001 -> -0.9510565162951538 +sinpi10115 sinpi 1.4 -> -0.9510565162951535 +sinpi10116 sinpi -1.399999976158142 -> 0.9510564931493437 +sinpi10117 sinpi -1.4000000953674316 -> 0.9510566088783399 +sinpi10118 sinpi -1.4 -> 0.9510565162951535 +sinpi10119 sinpi -1.4000000000000001 -> 0.9510565162951538 +sinpi10120 sinpi 1.5 -> -1.0 +sinpi10121 sinpi -1.5 -> 1.0 +sinpi10122 sinpi 2.5 -> 1.0 +sinpi10123 sinpi -2.5 -> -1.0 +sinpi10124 sinpi 3.5 -> -1.0 +sinpi10125 sinpi -3.5 -> 1.0 +sinpi10126 sinpi 4.5 -> 1.0 +sinpi10127 sinpi -4.5 -> -1.0 +sinpi10128 sinpi 0.24000000953674316 -> 0.6845471277689983 +sinpi10129 sinpi 0.23999999463558197 -> 0.6845470936435142 +sinpi10130 sinpi 0.24000000000000002 -> 0.6845471059286887 +sinpi10131 sinpi 0.24 -> 0.6845471059286886 +sinpi10132 sinpi -0.23999999463558197 -> -0.6845470936435142 +sinpi10133 sinpi -0.24000000953674316 -> -0.6845471277689983 +sinpi10134 sinpi -0.24 -> -0.6845471059286886 +sinpi10135 sinpi -0.24000000000000002 -> -0.6845471059286887 +sinpi10136 sinpi 0.49000000953674316 -> 0.9995065613068151 +sinpi10137 sinpi 0.4899999797344208 -> 0.9995065583659261 +sinpi10138 sinpi 0.49000000000000005 -> 0.9995065603657316 +sinpi10139 sinpi 0.49 -> 0.9995065603657316 +sinpi10140 sinpi -0.4899999797344208 -> -0.9995065583659261 +sinpi10141 sinpi -0.49000000953674316 -> -0.9995065613068151 +sinpi10142 sinpi -0.49 -> -0.9995065603657316 +sinpi10143 sinpi -0.49000000000000005 -> -0.9995065603657316 +sinpi10144 sinpi 0.5100000500679016 -> 0.9995065554250282 +sinpi10145 sinpi 0.5099999904632568 -> 0.9995065613068151 +sinpi10146 sinpi 0.51 -> 0.9995065603657316 +sinpi10147 sinpi 0.5099999999999999 -> 0.9995065603657316 +sinpi10148 sinpi -0.5099999904632568 -> -0.9995065613068151 +sinpi10149 sinpi -0.5100000500679016 -> -0.9995065554250282 +sinpi10150 sinpi -0.5099999999999999 -> -0.9995065603657316 +sinpi10151 sinpi -0.51 -> -0.9995065603657316 +sinpi10152 sinpi 0.7600000500679016 -> 0.684546991267053 +sinpi10153 sinpi 0.7599999904632568 -> 0.6845471277689983 +sinpi10154 sinpi 0.76 -> 0.6845471059286886 +sinpi10155 sinpi 0.7599999999999999 -> 0.684547105928689 +sinpi10156 sinpi -0.7599999904632568 -> -0.6845471277689983 +sinpi10157 sinpi -0.7600000500679016 -> -0.684546991267053 +sinpi10158 sinpi -0.7599999999999999 -> -0.684547105928689 +sinpi10159 sinpi -0.76 -> -0.6845471059286886 +sinpi10160 sinpi 1.2400000095367432 -> -0.6845471277689983 +sinpi10161 sinpi 1.2399998903274536 -> -0.6845468547650836 +sinpi10162 sinpi 1.2400000000000002 -> -0.6845471059286892 +sinpi10163 sinpi 1.24 -> -0.6845471059286886 +sinpi10164 sinpi -1.2399998903274536 -> 0.6845468547650836 +sinpi10165 sinpi -1.2400000095367432 -> 0.6845471277689983 +sinpi10166 sinpi -1.24 -> 0.6845471059286886 +sinpi10167 sinpi -1.2400000000000002 -> 0.6845471059286892 +sinpi10168 sinpi 1.4900000095367432 -> -0.9995065613068151 +sinpi10169 sinpi 1.4899998903274536 -> -0.9995065495432062 +sinpi10170 sinpi 1.4900000000000002 -> -0.9995065603657316 +sinpi10171 sinpi 1.49 -> -0.9995065603657316 +sinpi10172 sinpi -1.4899998903274536 -> 0.9995065495432062 +sinpi10173 sinpi -1.4900000095367432 -> 0.9995065613068151 +sinpi10174 sinpi -1.49 -> 0.9995065603657316 +sinpi10175 sinpi -1.4900000000000002 -> 0.9995065603657316 +sinpi10176 sinpi 1.5100001096725464 -> -0.9995065495432062 +sinpi10177 sinpi 1.5099999904632568 -> -0.9995065613068151 +sinpi10178 sinpi 1.51 -> -0.9995065603657316 +sinpi10179 sinpi 1.5099999999999998 -> -0.9995065603657316 +sinpi10180 sinpi -1.5099999904632568 -> 0.9995065613068151 +sinpi10181 sinpi -1.5100001096725464 -> 0.9995065495432062 +sinpi10182 sinpi -1.5099999999999998 -> 0.9995065603657316 +sinpi10183 sinpi -1.51 -> 0.9995065603657316 +sinpi10184 sinpi 1.7600001096725464 -> -0.6845468547650836 +sinpi10185 sinpi 1.7599999904632568 -> -0.6845471277689983 +sinpi10186 sinpi 1.76 -> -0.6845471059286886 +sinpi10187 sinpi 1.7599999999999998 -> -0.6845471059286892 +sinpi10188 sinpi -1.7599999904632568 -> 0.6845471277689983 +sinpi10189 sinpi -1.7600001096725464 -> 0.6845468547650836 +sinpi10190 sinpi -1.7599999999999998 -> 0.6845471059286892 +sinpi10191 sinpi -1.76 -> 0.6845471059286886 +sinpi10192 sinpi 100.24000549316406 -> 0.6845596858452854 +sinpi10193 sinpi 100.23999786376953 -> 0.684542213683844 +sinpi10194 sinpi 100.24000000000001 -> 0.6845471059287095 +sinpi10195 sinpi 100.24 -> 0.684547105928677 +sinpi10196 sinpi -100.23999786376953 -> -0.684542213683844 +sinpi10197 sinpi -100.24000549316406 -> -0.6845596858452854 +sinpi10198 sinpi -100.24 -> -0.684547105928677 +sinpi10199 sinpi -100.24000000000001 -> -0.6845471059287095 +sinpi10200 sinpi 100.49000549316406 -> 0.9995071022812839 +sinpi10201 sinpi 100.48999786376953 -> 0.9995063495404061 +sinpi10202 sinpi 100.49000000000001 -> 0.9995065603657325 +sinpi10203 sinpi 100.49 -> 0.999506560365731 +sinpi10204 sinpi -100.48999786376953 -> -0.9995063495404061 +sinpi10205 sinpi -100.49000549316406 -> -0.9995071022812839 +sinpi10206 sinpi -100.49 -> -0.999506560365731 +sinpi10207 sinpi -100.49000000000001 -> -0.9995065603657325 +sinpi10208 sinpi 100.51000213623047 -> 0.9995063495404061 +sinpi10209 sinpi 100.50999450683594 -> 0.9995071022812839 +sinpi10210 sinpi 100.51 -> 0.999506560365731 +sinpi10211 sinpi 100.50999999999999 -> 0.9995065603657325 +sinpi10212 sinpi -100.50999450683594 -> -0.9995071022812839 +sinpi10213 sinpi -100.51000213623047 -> -0.9995063495404061 +sinpi10214 sinpi -100.50999999999999 -> -0.9995065603657325 +sinpi10215 sinpi -100.51 -> -0.999506560365731 +sinpi10216 sinpi 100.76000213623047 -> 0.684542213683844 +sinpi10217 sinpi 100.75999450683594 -> 0.6845596858452854 +sinpi10218 sinpi 100.76 -> 0.684547105928677 +sinpi10219 sinpi 100.75999999999999 -> 0.6845471059287095 +sinpi10220 sinpi -100.75999450683594 -> -0.6845596858452854 +sinpi10221 sinpi -100.76000213623047 -> -0.684542213683844 +sinpi10222 sinpi -100.75999999999999 -> -0.6845471059287095 +sinpi10223 sinpi -100.76 -> -0.684547105928677 +sinpi10224 sinpi 1234.56005859375 -> 0.9822527413662894 +sinpi10225 sinpi 1234.5599365234375 -> 0.9823245983107213 +sinpi10226 sinpi 1234.5600000000002 -> 0.9822872507285869 +sinpi10227 sinpi 1234.56 -> 0.9822872507287208 +sinpi10228 sinpi -1234.5599365234375 -> -0.9823245983107213 +sinpi10229 sinpi -1234.56005859375 -> -0.9822527413662894 +sinpi10230 sinpi -1234.56 -> -0.9822872507287208 +sinpi10231 sinpi -1234.5600000000002 -> -0.9822872507285869 +sinpi10232 sinpi 2097151.125 -> -0.3826834323650898 +sinpi10233 sinpi -2097151.125 -> 0.3826834323650898 +sinpi10234 sinpi 2097151.875 -> -0.3826834323650898 +sinpi10235 sinpi -2097151.875 -> 0.3826834323650898 +sinpi10236 sinpi 1125899906842624.0 -> 0.0 +sinpi10237 sinpi 1125899839733760.0 -> 0.0 +sinpi10238 sinpi 1125899906842623.1 -> -0.3826834323650898 +sinpi10239 sinpi -1125899839733760.0 -> -0.0 +sinpi10240 sinpi -1125899906842624.0 -> -0.0 +sinpi10241 sinpi -1125899906842623.1 -> 0.3826834323650898 +sinpi10242 sinpi 1125899906842624.0 -> 0.0 +sinpi10243 sinpi 1125899839733760.0 -> 0.0 +sinpi10244 sinpi 1125899906842623.9 -> -0.3826834323650898 +sinpi10245 sinpi -1125899839733760.0 -> -0.0 +sinpi10246 sinpi -1125899906842624.0 -> -0.0 +sinpi10247 sinpi -1125899906842623.9 -> 0.3826834323650898 +sinpi10248 sinpi 2.305843009213694e+18 -> 0.0 +sinpi10249 sinpi 2.3058428717747405e+18 -> 0.0 +sinpi10250 sinpi 2.3058430092136937e+18 -> 0.0 +sinpi10251 sinpi -2.3058428717747405e+18 -> -0.0 +sinpi10252 sinpi -2.305843009213694e+18 -> -0.0 +sinpi10253 sinpi -2.3058430092136937e+18 -> -0.0 +sinpi10254 sinpi 2.305843009213694e+18 -> 0.0 +sinpi10255 sinpi 2.3058428717747405e+18 -> 0.0 +sinpi10256 sinpi 2.3058430092136937e+18 -> 0.0 +sinpi10257 sinpi -2.3058428717747405e+18 -> -0.0 +sinpi10258 sinpi -2.305843009213694e+18 -> -0.0 +sinpi10259 sinpi -2.3058430092136937e+18 -> -0.0 +sinpi10260 sinpi 1.298074214633707e+33 -> 0.0 +sinpi10261 sinpi 1.2980741372624545e+33 -> 0.0 +sinpi10262 sinpi 1.2980742146337068e+33 -> 0.0 +sinpi10263 sinpi -1.2980741372624545e+33 -> -0.0 +sinpi10264 sinpi -1.298074214633707e+33 -> -0.0 +sinpi10265 sinpi -1.2980742146337068e+33 -> -0.0 +sinpi10266 sinpi 1.298074214633707e+33 -> 0.0 +sinpi10267 sinpi 1.2980741372624545e+33 -> 0.0 +sinpi10268 sinpi 1.2980742146337068e+33 -> 0.0 +sinpi10269 sinpi -1.2980741372624545e+33 -> -0.0 +sinpi10270 sinpi -1.298074214633707e+33 -> -0.0 +sinpi10271 sinpi -1.2980742146337068e+33 -> -0.0 +sinpi10272 sinpi 2.938735877055719e-39 -> 9.232311042199004e-39 +sinpi10273 sinpi -2.938735877055719e-39 -> -9.232311042199004e-39 +sinpi10274 sinpi 1.401298464324817e-45 -> 4.402308961009504e-45 +sinpi10275 sinpi 0.0 -> 0.0 +sinpi10276 sinpi 5.010420900022432e-293 -> 1.5740701490903231e-292 +sinpi10277 sinpi -0.0 -> -0.0 +sinpi10278 sinpi -1.401298464324817e-45 -> -4.402308961009504e-45 +sinpi10279 sinpi -5.010420900022432e-293 -> -1.5740701490903231e-292 +sinpi10280 sinpi 1.401298464324817e-45 -> 4.402308961009504e-45 +sinpi10281 sinpi 0.0 -> 0.0 +sinpi10282 sinpi 5.562684646268003e-309 -> 1.7475689218952297e-308 +sinpi10283 sinpi -0.0 -> -0.0 +sinpi10284 sinpi -1.401298464324817e-45 -> -4.402308961009504e-45 +sinpi10285 sinpi -5.562684646268003e-309 -> -1.7475689218952297e-308 +sinpi10286 sinpi 1.401298464324817e-45 -> 4.402308961009504e-45 +sinpi10287 sinpi 0.0 -> 0.0 +sinpi10288 sinpi 5e-324 -> 1.5e-323 +sinpi10289 sinpi -0.0 -> -0.0 +sinpi10290 sinpi -1.401298464324817e-45 -> -4.402308961009504e-45 +sinpi10291 sinpi -5e-324 -> -1.5e-323 +sinpi10292 sinpi 1.401298464324817e-45 -> 4.402308961009504e-45 +sinpi10293 sinpi 0.0 -> 0.0 +sinpi10294 sinpi 5e-324 -> 1.5e-323 +sinpi10295 sinpi -0.0 -> -0.0 +sinpi10296 sinpi -1.401298464324817e-45 -> -4.402308961009504e-45 +sinpi10297 sinpi -5e-324 -> -1.5e-323 +sinpi10298 sinpi 0.039865780621767044 -> 0.12491488475763585 +sinpi10299 sinpi -0.009947286918759346 -> -0.03124523733449905 +sinpi10300 sinpi -0.00994728785008192 -> -0.03124524025890666 +sinpi10301 sinpi -0.009947287497407154 -> -0.03124523915148717 +sinpi10302 sinpi 0.009947438724339008 -> 0.03124571401293662 +sinpi10303 sinpi 0.009947437793016434 -> 0.03124571108852905 +sinpi10304 sinpi 0.009947437917099084 -> 0.03124571147815586 +sinpi10305 sinpi 0.009947437917099082 -> 0.031245711478155854 +sinpi10306 sinpi -0.0049737198278307915 -> -0.015624765847977973 +sinpi10307 sinpi -0.004973720293492079 -> -0.015624767310717469 +sinpi10308 sinpi -0.004973719955059152 -> -0.015624766247628864 +sinpi10309 sinpi -0.004973719955059153 -> -0.015624766247628865 + +sinpi20001 sinpi inf -> nan invalid +sinpi20002 sinpi -inf -> nan invalid +sinpi20003 sinpi nan -> nan + +----------- +-- tanpi -- +----------- + +tanpi10000 tanpi 0.0 -> 0.0 +tanpi10001 tanpi -0.0 -> -0.0 +tanpi10002 tanpi 1.1754943508222875e-38 -> 3.6929244168796014e-38 +tanpi10003 tanpi 2.2250738585072014e-308 -> 6.990275687580919e-308 +tanpi10004 tanpi 2.004168360008973e-292 -> 6.296280596361293e-292 +tanpi10005 tanpi -1.1754943508222875e-38 -> -3.6929244168796014e-38 +tanpi10006 tanpi -2.2250738585072014e-308 -> -6.990275687580919e-308 +tanpi10007 tanpi -2.004168360008973e-292 -> -6.296280596361293e-292 +tanpi10008 tanpi 1.401298464324817e-45 -> 4.402308961009504e-45 +tanpi10009 tanpi 5e-324 -> 1.5e-323 +tanpi10010 tanpi -1.401298464324817e-45 -> -4.402308961009504e-45 +tanpi10011 tanpi -5e-324 -> -1.5e-323 +tanpi10012 tanpi 3.4028234663852886e+38 -> 0.0 +tanpi10013 tanpi 1.7976931348623157e+308 -> 0.0 +tanpi10014 tanpi -3.4028234663852886e+38 -> -0.0 +tanpi10015 tanpi -1.7976931348623157e+308 -> -0.0 +tanpi10016 tanpi 1.0 -> -0.0 +tanpi10017 tanpi -1.0 -> 0.0 +tanpi10018 tanpi 2.0 -> 0.0 +tanpi10019 tanpi -2.0 -> -0.0 +tanpi10020 tanpi 3.0 -> -0.0 +tanpi10021 tanpi -3.0 -> 0.0 +tanpi10022 tanpi 4.0 -> 0.0 +tanpi10023 tanpi -4.0 -> -0.0 +tanpi10024 tanpi 0.10000000149011612 -> 0.32491970140846616 +tanpi10025 tanpi 0.09999999403953552 -> 0.3249196755306671 +tanpi10026 tanpi 0.1 -> 0.32491969623290634 +tanpi10027 tanpi 0.09999999999999999 -> 0.3249196962329063 +tanpi10028 tanpi -0.09999999403953552 -> -0.3249196755306671 +tanpi10029 tanpi -0.10000000149011612 -> -0.32491970140846616 +tanpi10030 tanpi -0.09999999999999999 -> -0.3249196962329063 +tanpi10031 tanpi -0.1 -> -0.32491969623290634 +tanpi10032 tanpi 0.20000000298023224 -> 0.7265425423102566 +tanpi10033 tanpi 0.19999998807907104 -> 0.7265424707857802 +tanpi10034 tanpi 0.2 -> 0.7265425280053609 +tanpi10035 tanpi 0.19999999999999998 -> 0.7265425280053608 +tanpi10036 tanpi -0.19999998807907104 -> -0.7265424707857802 +tanpi10037 tanpi -0.20000000298023224 -> -0.7265425423102566 +tanpi10038 tanpi -0.19999999999999998 -> -0.7265425280053608 +tanpi10039 tanpi -0.2 -> -0.7265425280053609 +tanpi10040 tanpi 0.30000001192092896 -> 1.3763820288695117 +tanpi10041 tanpi 0.29999998211860657 -> 1.3763817578736872 +tanpi10042 tanpi 0.30000000000000004 -> 1.376381920471174 +tanpi10043 tanpi 0.3 -> 1.3763819204711734 +tanpi10044 tanpi -0.29999998211860657 -> -1.3763817578736872 +tanpi10045 tanpi -0.30000001192092896 -> -1.3763820288695117 +tanpi10046 tanpi -0.3 -> -1.3763819204711734 +tanpi10047 tanpi -0.30000000000000004 -> -1.376381920471174 +tanpi10048 tanpi 0.4000000059604645 -> 3.0776837332696907 +tanpi10049 tanpi 0.3999999761581421 -> 3.077682752797731 +tanpi10050 tanpi 0.4 -> 3.077683537175254 +tanpi10051 tanpi 0.39999999999999997 -> 3.0776835371752522 +tanpi10052 tanpi -0.3999999761581421 -> -3.077682752797731 +tanpi10053 tanpi -0.4000000059604645 -> -3.0776837332696907 +tanpi10054 tanpi -0.39999999999999997 -> -3.0776835371752522 +tanpi10055 tanpi -0.4 -> -3.077683537175254 +tanpi10056 tanpi 0.6000000238418579 -> -3.077682752797731 +tanpi10057 tanpi 0.5999999642372131 -> -3.077684713742215 +tanpi10058 tanpi 0.6000000000000001 -> -3.0776835371752504 +tanpi10059 tanpi 0.6 -> -3.077683537175254 +tanpi10060 tanpi -0.5999999642372131 -> 3.077684713742215 +tanpi10061 tanpi -0.6000000238418579 -> 3.077682752797731 +tanpi10062 tanpi -0.6 -> 3.077683537175254 +tanpi10063 tanpi -0.6000000000000001 -> 3.0776835371752504 +tanpi10064 tanpi 0.7000000476837158 -> -1.3763814868779325 +tanpi10065 tanpi 0.699999988079071 -> -1.3763820288695117 +tanpi10066 tanpi 0.7000000000000001 -> -1.376381920471173 +tanpi10067 tanpi 0.7 -> -1.376381920471174 +tanpi10068 tanpi -0.699999988079071 -> 1.3763820288695117 +tanpi10069 tanpi -0.7000000476837158 -> 1.3763814868779325 +tanpi10070 tanpi -0.7 -> 1.376381920471174 +tanpi10071 tanpi -0.7000000000000001 -> 1.376381920471173 +tanpi10072 tanpi 0.800000011920929 -> -0.7265424707857802 +tanpi10073 tanpi 0.7999999523162842 -> -0.7265427568837151 +tanpi10074 tanpi 0.8 -> -0.7265425280053607 +tanpi10075 tanpi 0.7999999999999999 -> -0.7265425280053612 +tanpi10076 tanpi -0.7999999523162842 -> 0.7265427568837151 +tanpi10077 tanpi -0.800000011920929 -> 0.7265424707857802 +tanpi10078 tanpi -0.7999999999999999 -> 0.7265425280053612 +tanpi10079 tanpi -0.8 -> 0.7265425280053607 +tanpi10080 tanpi 0.9000000357627869 -> -0.3249195720194747 +tanpi10081 tanpi 0.8999999761581421 -> -0.3249197790418658 +tanpi10082 tanpi 0.9 -> -0.32491969623290623 +tanpi10083 tanpi 0.8999999999999999 -> -0.3249196962329066 +tanpi10084 tanpi -0.8999999761581421 -> 0.3249197790418658 +tanpi10085 tanpi -0.9000000357627869 -> 0.3249195720194747 +tanpi10086 tanpi -0.8999999999999999 -> 0.3249196962329066 +tanpi10087 tanpi -0.9 -> 0.32491969623290623 +tanpi10088 tanpi 1.100000023841858 -> 0.3249197790418658 +tanpi10089 tanpi 1.0999999046325684 -> 0.3249193649971088 +tanpi10090 tanpi 1.1 -> 0.3249196962329066 +tanpi10091 tanpi 1.0999999999999999 -> 0.32491969623290584 +tanpi10092 tanpi -1.0999999046325684 -> -0.3249193649971088 +tanpi10093 tanpi -1.100000023841858 -> -0.3249197790418658 +tanpi10094 tanpi -1.0999999999999999 -> -0.32491969623290584 +tanpi10095 tanpi -1.1 -> -0.3249196962329066 +tanpi10096 tanpi 1.2000000476837158 -> 0.7265427568837151 +tanpi10097 tanpi 1.1999999284744263 -> 0.7265421846879231 +tanpi10098 tanpi 1.2000000000000002 -> 0.7265425280053618 +tanpi10099 tanpi 1.2 -> 0.7265425280053607 +tanpi10100 tanpi -1.1999999284744263 -> -0.7265421846879231 +tanpi10101 tanpi -1.2000000476837158 -> -0.7265427568837151 +tanpi10102 tanpi -1.2 -> -0.7265425280053607 +tanpi10103 tanpi -1.2000000000000002 -> -0.7265425280053618 +tanpi10104 tanpi 1.3000000715255737 -> 1.3763825708613702 +tanpi10105 tanpi 1.2999999523162842 -> 1.3763814868779325 +tanpi10106 tanpi 1.3 -> 1.376381920471174 +tanpi10107 tanpi 1.2999999999999998 -> 1.376381920471172 +tanpi10108 tanpi -1.2999999523162842 -> -1.3763814868779325 +tanpi10109 tanpi -1.3000000715255737 -> -1.3763825708613702 +tanpi10110 tanpi -1.2999999999999998 -> -1.376381920471172 +tanpi10111 tanpi -1.3 -> -1.376381920471174 +tanpi10112 tanpi 1.4000000953674316 -> 3.0776866746889597 +tanpi10113 tanpi 1.399999976158142 -> 3.077682752797731 +tanpi10114 tanpi 1.4000000000000001 -> 3.077683537175258 +tanpi10115 tanpi 1.4 -> 3.0776835371752504 +tanpi10116 tanpi -1.399999976158142 -> -3.077682752797731 +tanpi10117 tanpi -1.4000000953674316 -> -3.0776866746889597 +tanpi10118 tanpi -1.4 -> -3.0776835371752504 +tanpi10119 tanpi -1.4000000000000001 -> -3.077683537175258 +tanpi10120 tanpi 0.24000000953674316 -> 0.9390625621984302 +tanpi10121 tanpi 0.23999999463558197 -> 0.9390624741032162 +tanpi10122 tanpi 0.24000000000000002 -> 0.9390625058174925 +tanpi10123 tanpi 0.24 -> 0.9390625058174923 +tanpi10124 tanpi -0.23999999463558197 -> -0.9390624741032162 +tanpi10125 tanpi -0.24000000953674316 -> -0.9390625621984302 +tanpi10126 tanpi -0.24 -> -0.9390625058174923 +tanpi10127 tanpi -0.24000000000000002 -> -0.9390625058174925 +tanpi10128 tanpi 0.49000000953674316 -> 31.820546320188043 +tanpi10129 tanpi 0.4899999797344208 -> 31.82045142533627 +tanpi10130 tanpi 0.49000000000000005 -> 31.820515953774105 +tanpi10131 tanpi 0.49 -> 31.82051595377393 +tanpi10132 tanpi -0.4899999797344208 -> -31.82045142533627 +tanpi10133 tanpi -0.49000000953674316 -> -31.820546320188043 +tanpi10134 tanpi -0.49 -> -31.82051595377393 +tanpi10135 tanpi -0.49000000000000005 -> -31.820515953774105 +tanpi10136 tanpi 0.5100000500679016 -> -31.820356531049924 +tanpi10137 tanpi 0.5099999904632568 -> -31.820546320188043 +tanpi10138 tanpi 0.51 -> -31.82051595377393 +tanpi10139 tanpi 0.5099999999999999 -> -31.820515953774283 +tanpi10140 tanpi -0.5099999904632568 -> 31.820546320188043 +tanpi10141 tanpi -0.5100000500679016 -> 31.820356531049924 +tanpi10142 tanpi -0.5099999999999999 -> 31.820515953774283 +tanpi10143 tanpi -0.51 -> 31.82051595377393 +tanpi10144 tanpi 0.7600000500679016 -> -0.9390622098176208 +tanpi10145 tanpi 0.7599999904632568 -> -0.9390625621984302 +tanpi10146 tanpi 0.76 -> -0.9390625058174923 +tanpi10147 tanpi 0.7599999999999999 -> -0.939062505817493 +tanpi10148 tanpi -0.7599999904632568 -> 0.9390625621984302 +tanpi10149 tanpi -0.7600000500679016 -> 0.9390622098176208 +tanpi10150 tanpi -0.7599999999999999 -> 0.939062505817493 +tanpi10151 tanpi -0.76 -> 0.9390625058174923 +tanpi10152 tanpi 1.2400000095367432 -> 0.9390625621984302 +tanpi10153 tanpi 1.2399998903274536 -> 0.9390618574369353 +tanpi10154 tanpi 1.2400000000000002 -> 0.9390625058174936 +tanpi10155 tanpi 1.24 -> 0.9390625058174923 +tanpi10156 tanpi -1.2399998903274536 -> -0.9390618574369353 +tanpi10157 tanpi -1.2400000095367432 -> -0.9390625621984302 +tanpi10158 tanpi -1.24 -> -0.9390625058174923 +tanpi10159 tanpi -1.2400000000000002 -> -0.9390625058174936 +tanpi10160 tanpi 1.4900000095367432 -> 31.820546320188043 +tanpi10161 tanpi 1.4899998903274536 -> 31.820166744173495 +tanpi10162 tanpi 1.4900000000000002 -> 31.82051595377464 +tanpi10163 tanpi 1.49 -> 31.82051595377393 +tanpi10164 tanpi -1.4899998903274536 -> -31.820166744173495 +tanpi10165 tanpi -1.4900000095367432 -> -31.820546320188043 +tanpi10166 tanpi -1.49 -> -31.82051595377393 +tanpi10167 tanpi -1.4900000000000002 -> -31.82051595377464 +tanpi10168 tanpi 1.5100001096725464 -> -31.820166744173495 +tanpi10169 tanpi 1.5099999904632568 -> -31.820546320188043 +tanpi10170 tanpi 1.51 -> -31.82051595377393 +tanpi10171 tanpi 1.5099999999999998 -> -31.82051595377464 +tanpi10172 tanpi -1.5099999904632568 -> 31.820546320188043 +tanpi10173 tanpi -1.5100001096725464 -> 31.820166744173495 +tanpi10174 tanpi -1.5099999999999998 -> 31.82051595377464 +tanpi10175 tanpi -1.51 -> 31.82051595377393 +tanpi10176 tanpi 1.7600001096725464 -> -0.9390618574369353 +tanpi10177 tanpi 1.7599999904632568 -> -0.9390625621984302 +tanpi10178 tanpi 1.76 -> -0.9390625058174923 +tanpi10179 tanpi 1.7599999999999998 -> -0.9390625058174936 +tanpi10180 tanpi -1.7599999904632568 -> 0.9390625621984302 +tanpi10181 tanpi -1.7600001096725464 -> 0.9390618574369353 +tanpi10182 tanpi -1.7599999999999998 -> 0.9390625058174936 +tanpi10183 tanpi -1.76 -> 0.9390625058174923 +tanpi10184 tanpi 100.24000549316406 -> 0.9390949817630689 +tanpi10185 tanpi 100.23999786376953 -> 0.9390498765673647 +tanpi10186 tanpi 100.24000000000001 -> 0.9390625058175461 +tanpi10187 tanpi 100.24 -> 0.9390625058174621 +tanpi10188 tanpi -100.23999786376953 -> -0.9390498765673647 +tanpi10189 tanpi -100.24000549316406 -> -0.9390949817630689 +tanpi10190 tanpi -100.24 -> -0.9390625058174621 +tanpi10191 tanpi -100.24000000000001 -> -0.9390625058175461 +tanpi10192 tanpi 100.49000549316406 -> 31.83801660184476 +tanpi10193 tanpi 100.48999786376953 -> 31.81371533579395 +tanpi10194 tanpi 100.49000000000001 -> 31.820515953802918 +tanpi10195 tanpi 100.49 -> 31.820515953757667 +tanpi10196 tanpi -100.48999786376953 -> -31.81371533579395 +tanpi10197 tanpi -100.49000549316406 -> -31.83801660184476 +tanpi10198 tanpi -100.49 -> -31.820515953757667 +tanpi10199 tanpi -100.49000000000001 -> -31.820515953802918 +tanpi10200 tanpi 100.51000213623047 -> -31.81371533579395 +tanpi10201 tanpi 100.50999450683594 -> -31.83801660184476 +tanpi10202 tanpi 100.51 -> -31.820515953757667 +tanpi10203 tanpi 100.50999999999999 -> -31.820515953802918 +tanpi10204 tanpi -100.50999450683594 -> 31.83801660184476 +tanpi10205 tanpi -100.51000213623047 -> 31.81371533579395 +tanpi10206 tanpi -100.50999999999999 -> 31.820515953802918 +tanpi10207 tanpi -100.51 -> 31.820515953757667 +tanpi10208 tanpi 100.76000213623047 -> -0.9390498765673647 +tanpi10209 tanpi 100.75999450683594 -> -0.9390949817630689 +tanpi10210 tanpi 100.76 -> -0.9390625058174621 +tanpi10211 tanpi 100.75999999999999 -> -0.9390625058175461 +tanpi10212 tanpi -100.75999450683594 -> 0.9390949817630689 +tanpi10213 tanpi -100.76000213623047 -> 0.9390498765673647 +tanpi10214 tanpi -100.75999999999999 -> 0.9390625058175461 +tanpi10215 tanpi -100.76 -> 0.9390625058174621 +tanpi10216 tanpi 1234.56005859375 -> -5.236946012446678 +tanpi10217 tanpi 1234.5599365234375 -> -5.247869032587774 +tanpi10218 tanpi 1234.5600000000002 -> -5.2421835810977155 +tanpi10219 tanpi 1234.56 -> -5.242183581118059 +tanpi10220 tanpi -1234.5599365234375 -> 5.247869032587774 +tanpi10221 tanpi -1234.56005859375 -> 5.236946012446678 +tanpi10222 tanpi -1234.56 -> 5.242183581118059 +tanpi10223 tanpi -1234.5600000000002 -> 5.2421835810977155 +tanpi10224 tanpi 2097151.375 -> 2.414213562373095 +tanpi10225 tanpi -2097151.375 -> -2.414213562373095 +tanpi10226 tanpi 2097151.625 -> -2.414213562373095 +tanpi10227 tanpi -2097151.625 -> 2.414213562373095 +tanpi10228 tanpi 1125899906842624.0 -> 0.0 +tanpi10229 tanpi 1125899839733760.0 -> 0.0 +tanpi10230 tanpi 1125899906842623.4 -> 2.414213562373095 +tanpi10231 tanpi -1125899839733760.0 -> -0.0 +tanpi10232 tanpi -1125899906842624.0 -> -0.0 +tanpi10233 tanpi -1125899906842623.4 -> -2.414213562373095 +tanpi10234 tanpi 1125899906842624.0 -> 0.0 +tanpi10235 tanpi 1125899839733760.0 -> 0.0 +tanpi10236 tanpi 1125899906842623.6 -> -2.414213562373095 +tanpi10237 tanpi -1125899839733760.0 -> -0.0 +tanpi10238 tanpi -1125899906842624.0 -> -0.0 +tanpi10239 tanpi -1125899906842623.6 -> 2.414213562373095 +tanpi10240 tanpi 2.305843009213694e+18 -> 0.0 +tanpi10241 tanpi 2.3058428717747405e+18 -> 0.0 +tanpi10242 tanpi 2.3058430092136937e+18 -> 0.0 +tanpi10243 tanpi -2.3058428717747405e+18 -> -0.0 +tanpi10244 tanpi -2.305843009213694e+18 -> -0.0 +tanpi10245 tanpi -2.3058430092136937e+18 -> -0.0 +tanpi10246 tanpi 2.305843009213694e+18 -> 0.0 +tanpi10247 tanpi 2.3058428717747405e+18 -> 0.0 +tanpi10248 tanpi 2.3058430092136937e+18 -> 0.0 +tanpi10249 tanpi -2.3058428717747405e+18 -> -0.0 +tanpi10250 tanpi -2.305843009213694e+18 -> -0.0 +tanpi10251 tanpi -2.3058430092136937e+18 -> -0.0 +tanpi10252 tanpi 1.298074214633707e+33 -> 0.0 +tanpi10253 tanpi 1.2980741372624545e+33 -> 0.0 +tanpi10254 tanpi 1.2980742146337068e+33 -> 0.0 +tanpi10255 tanpi -1.2980741372624545e+33 -> -0.0 +tanpi10256 tanpi -1.298074214633707e+33 -> -0.0 +tanpi10257 tanpi -1.2980742146337068e+33 -> -0.0 +tanpi10258 tanpi 1.298074214633707e+33 -> 0.0 +tanpi10259 tanpi 1.2980741372624545e+33 -> 0.0 +tanpi10260 tanpi 1.2980742146337068e+33 -> 0.0 +tanpi10261 tanpi -1.2980741372624545e+33 -> -0.0 +tanpi10262 tanpi -1.298074214633707e+33 -> -0.0 +tanpi10263 tanpi -1.2980742146337068e+33 -> -0.0 +tanpi10264 tanpi 0.48004481196403503 -> 15.930332115434627 +tanpi10265 tanpi -0.4950249195098877 -> -63.97564155226383 +tanpi10266 tanpi -0.4950249493122101 -> -63.97602485152189 +tanpi10267 tanpi -0.49502493342943843 -> -63.97582057644752 +tanpi10268 tanpi -3.659269094467163 -> 1.828928514937406 +tanpi10269 tanpi -3.659269332885742 -> 1.8289252604911337 +tanpi10270 tanpi -3.659269111498701 -> 1.8289282824542836 +tanpi10271 tanpi -3.6592691114987015 -> 1.8289282824542774 +tanpi10272 tanpi -1.6591962575912476 -> 1.829923165916756 +tanpi10273 tanpi -1.659196376800537 -> 1.8299215373295707 +tanpi10274 tanpi -1.6591963470121214 -> 1.8299219442861712 +tanpi10275 tanpi -1.6591963470121216 -> 1.829921944286168 + +tanpi20001 tanpi inf -> nan invalid +tanpi20002 tanpi -inf -> nan invalid +tanpi20003 tanpi nan -> nan diff --git a/Lib/test/memory_watchdog.py b/Lib/test/memory_watchdog.py index fee062ecc9b3001..4a3f66e1f822bab 100644 --- a/Lib/test/memory_watchdog.py +++ b/Lib/test/memory_watchdog.py @@ -1,21 +1,40 @@ """Memory watchdog: periodically read the memory usage of the main test process and print it out, until terminated.""" -# stdin should refer to the process' /proc//statm: we don't pass the -# process' PID to avoid a race condition in case of - unlikely - PID recycling. -# If the process crashes, reading from the /proc entry will fail with ESRCH. import sys import time -from test.support import get_pagesize - - -while True: - page_size = get_pagesize() - sys.stdin.seek(0) - statm = sys.stdin.read() - data = int(statm.split()[5]) - sys.stdout.write(" ... process data size: {data:.1f}G\n" - .format(data=data * page_size / (1024 ** 3))) - sys.stdout.flush() - time.sleep(1) +from test.libregrtest.utils import get_process_memory_usage + + +ONE_GIB = (1024 ** 3) + + +def watchdog(pid): + while True: + mem = get_process_memory_usage(pid) + if mem is None: + # get_process_memory_usage() is not supported on the platform, + # or something went wrong. Exit since the next call is likely to + # fail the same way. + return + + # Prefer sys.stdout.write() to print() to use a single write() syscall. + # print(msg) calls write(msg.encode()) and then write(b"\n"). + sys.stdout.write(f" ... process data size: {mem / ONE_GIB:.1f} GiB\n") + sys.stdout.flush() + time.sleep(1) + +def main(): + if len(sys.argv) != 2: + print(f"usage: python {sys.argv[0]} pid") + sys.exit(1) + pid = int(sys.argv[1]) + + try: + watchdog(pid) + except KeyboardInterrupt: + pass + +if __name__ == "__main__": + main() diff --git a/Lib/test/mime.types2 b/Lib/test/mime.types2 new file mode 100644 index 000000000000000..b05c318d1c05c2b --- /dev/null +++ b/Lib/test/mime.types2 @@ -0,0 +1,3 @@ +# MIME type Extensions +testing/test2 test test2 +testing/test3 test test3 diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 7f735d75b318e7f..a2767f3975bed25 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -8,6 +8,9 @@ import warnings +MS_WINDOWS = (sys.platform == "win32") + + def normalize_text(text): if text is None: return None @@ -16,6 +19,15 @@ def normalize_text(text): return text.strip() +def read_first_line(filename): + # Get the first line of a text file and strip trailing spaces + try: + with open(filename, encoding="utf-8") as fp: + return fp.readline().rstrip() + except OSError: + return '' + + class PythonInfo: def __init__(self): self.info = {} @@ -897,8 +909,30 @@ def collect_subprocess(info_add): copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',)) +def winreg_query(path): + try: + import winreg + except ImportError: + return None + + key, path = path.split('\\', 1) + sub_key, value = path.rsplit('\\', 1) + if key == "HKEY_LOCAL_MACHINE": + key = winreg.HKEY_LOCAL_MACHINE + else: + raise ValueError(f"unknown key {key!r}") + + try: + access = winreg.KEY_READ | winreg.KEY_WOW64_64KEY + with winreg.OpenKey(key, sub_key, access=access) as key_handle: + result, _ = winreg.QueryValueEx(key_handle, value) + return result + except OSError: + return None + + def collect_windows(info_add): - if sys.platform != "win32": + if not MS_WINDOWS: # Code specific to Windows return @@ -990,19 +1024,10 @@ def collect_windows(info_add): info_add('windows.ver', line) # windows.developer_mode: get AllowDevelopmentWithoutDevLicense registry - import winreg - try: - key = winreg.OpenKey( - winreg.HKEY_LOCAL_MACHINE, - r"SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock") - subkey = "AllowDevelopmentWithoutDevLicense" - try: - value, value_type = winreg.QueryValueEx(key, subkey) - finally: - winreg.CloseKey(key) - except OSError: - pass - else: + value = winreg_query(r"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows" + r"\CurrentVersion\AppModelUnlock" + r"\AllowDevelopmentWithoutDevLicense") + if value is not None: info_add('windows.developer_mode', "enabled" if value else "disabled") @@ -1015,14 +1040,9 @@ def collect_fips(info_add): if _hashlib is not None: call_func(info_add, 'fips.openssl_fips_mode', _hashlib, 'get_fips_mode') - try: - with open("/proc/sys/crypto/fips_enabled", encoding="utf-8") as fp: - line = fp.readline().rstrip() - - if line: - info_add('fips.linux_crypto_fips_enabled', line) - except OSError: - pass + fips_enabled = read_first_line("/proc/sys/crypto/fips_enabled") + if fips_enabled: + info_add('fips.linux_crypto_fips_enabled', fips_enabled) def collect_tempfile(info_add): @@ -1040,6 +1060,99 @@ def collect_libregrtest_utils(info_add): info_add('libregrtests.build_info', ' '.join(utils.get_build_info())) +def uptime_boottime(): + # Use CLOCK_BOOTTIME + import time + try: + return time.clock_gettime(time.CLOCK_BOOTTIME) + except (AttributeError, OSError): + return None + + +def uptime_linux(): + # Parse the first member of /proc/uptime + line = read_first_line("/proc/uptime") + if not line: + return + try: + parts = line.split() + if not parts: + return + return float(parts[0]) + except ValueError: + return + + +def uptime_bsd(): + # Get sysctlbyname("kern.boottime") + try: + import _testcapi + except ImportError: + return None + try: + return _testcapi.uptime_bsd() + except (AttributeError, OSError): + return None + + +def uptime_windows(): + try: + import _winapi + except ImportError: + return None + else: + return _winapi.GetTickCount64() / 1000. + + +def get_uptime(): + for func in (uptime_boottime, uptime_linux, uptime_bsd, uptime_windows): + uptime = func() + if uptime is not None: + return uptime + return None + + +def get_machine_id(): + if MS_WINDOWS: + machine_guid = winreg_query(r"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft" + r"\Cryptography\MachineGuid") + if machine_guid: + return machine_guid + + for filename in ( + # https://www.freedesktop.org/software/systemd/man/latest/machine-id.html + "/etc/machine-id", + # BSD + "/etc/hostid", + ): + machine_id = read_first_line(filename) + if machine_id: + return machine_id + + return None + + +def collect_linux(info_add): + boot_id = read_first_line("/proc/sys/kernel/random/boot_id") + if boot_id: + info_add('system.boot_id', boot_id) + + machine_id = get_machine_id() + if machine_id: + info_add('system.machine_id', machine_id) + + uptime = get_uptime() + if uptime is not None: + # truncate microseconds + uptime = int(uptime) + try: + import datetime + uptime = str(datetime.timedelta(seconds=uptime)) + except ImportError: + uptime = f'{uptime} sec' + info_add('system.uptime', uptime) + + def collect_info(info): error = False info_add = info.add @@ -1081,6 +1194,7 @@ def collect_info(info): collect_zlib, collect_zstd, collect_libregrtest_utils, + collect_linux, # Collecting from tests should be last as they have side effects. collect_test_socket, diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2cac70f4ab2afbe..e9966e1f7a6d495 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -73,6 +73,7 @@ "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", "reset_code", "on_github_actions", "requires_root_user", "requires_non_root_user", + "skip_if_double_rounding", ] @@ -324,16 +325,6 @@ def requires(resource, msg=None): if resource == 'gui' and not _is_gui_available(): raise ResourceDenied(_is_gui_available.reason) -def _get_kernel_version(sysname="Linux"): - import platform - if platform.system() != sysname: - return None - version_txt = platform.release().split('-', 1)[0] - try: - return tuple(map(int, version_txt.split('.'))) - except ValueError: - return None - def _requires_unix_version(sysname, min_version): """Decorator raising SkipTest if the OS is `sysname` and the version is less than `min_version`. @@ -524,6 +515,15 @@ def dec(*args, **kwargs): float.__getformat__("double").startswith("IEEE"), "test requires IEEE 754 doubles") +# detect evidence of double-rounding: +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) +skip_if_double_rounding = unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "accuracy not guaranteed on " + "machines with double rounding") +del x, y, HAVE_DOUBLE_ROUNDING + + def requires_zlib(reason='requires zlib'): try: import zlib @@ -1241,29 +1241,20 @@ class _MemoryWatchdog: """ def __init__(self): - self.procfile = '/proc/{pid}/statm'.format(pid=os.getpid()) self.started = False def start(self): - try: - f = open(self.procfile, 'r') - except OSError as e: - logging.getLogger(__name__).warning('/proc not available for stats: %s', e, exc_info=e) - sys.stderr.flush() - return - import subprocess - with f: - watchdog_script = findfile("memory_watchdog.py") - self.mem_watchdog = subprocess.Popen([sys.executable, watchdog_script], - stdin=f, - stderr=subprocess.DEVNULL) + watchdog_script = findfile("memory_watchdog.py") + cmd = [sys.executable, watchdog_script, str(os.getpid())] + self.mem_watchdog = subprocess.Popen(cmd) self.started = True def stop(self): - if self.started: - self.mem_watchdog.terminate() - self.mem_watchdog.wait() + if not self.started: + return + self.mem_watchdog.terminate() + self.mem_watchdog.wait() def bigmemtest(size, memuse, dry_run=True): @@ -1296,8 +1287,8 @@ def wrapper(self): if real_max_memuse and verbose: print() - print(" ... expected peak memory use: {peak:.1f}G" - .format(peak=size * memuse / (1024 ** 3))) + peak = (size * memuse) / (1024 ** 3) + print(f" ... expected peak memory use: {peak:.1f} GiB") watchdog = _MemoryWatchdog() watchdog.start() else: @@ -2815,6 +2806,10 @@ def exceeds_recursion_limit(): is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') +# Cygwin uses the newlib C library +skip_on_newlib = unittest.skipIf(sys.platform == 'cygwin', + 'the test fails on newlib C library') + Py_TRACE_REFS = hasattr(sys, 'getobjects') _JIT_ENABLED = sys._jit.is_enabled() @@ -3164,7 +3159,7 @@ def in_systemd_nspawn_sync_suppressed() -> bool: with open("/run/systemd/container", "rb") as fp: if fp.read().rstrip() != b"systemd-nspawn": return False - except FileNotFoundError: + except (FileNotFoundError, PermissionError): return False # If systemd-nspawn is used, O_SYNC flag will immediately @@ -3323,3 +3318,18 @@ def control_characters_c0() -> list[str]: _ROOT_IN_POSIX = hasattr(os, 'geteuid') and os.geteuid() == 0 requires_root_user = unittest.skipUnless(_ROOT_IN_POSIX, "test needs root privilege") requires_non_root_user = unittest.skipIf(_ROOT_IN_POSIX, "test needs non-root account") + + +STATUS_DLL_INIT_FAILED = 0xC0000142 +def skip_on_low_desktop_heap_memory_subprocess(returncode): + if sys.platform not in ('win32', 'cygwin'): + return + # On Windows, STATUS_DLL_INIT_FAILED is a generic error code that could + # come from any of the DLLs being loaded when a new Python process is + # created. In practice, it's likely a memory allocation failure in the + # desktop heap memory which caused the DLL init failure, especially on + # process created with CREATE_NEW_CONSOLE creation flag. See the article: + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/desktop-heap-limitation-out-of-memory + if returncode == STATUS_DLL_INIT_FAILED: + raise unittest.SkipTest('gh-150436: DLL init failed, likely because ' + 'of low desktop heap memory') diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index e8a58ed77061f55..e8a3d176ad6943f 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -325,7 +325,7 @@ def ready_to_import(name=None, source=""): sys.modules.pop(name, None) -def ensure_lazy_imports(imported_module, modules_to_block): +def ensure_lazy_imports(imported_module, modules_to_block, *, additional_code=None): """Test that when imported_module is imported, none of the modules in modules_to_block are imported as a side effect.""" modules_to_block = frozenset(modules_to_block) @@ -343,6 +343,16 @@ def ensure_lazy_imports(imported_module, modules_to_block): raise AssertionError(f'unexpectedly imported after importing {imported_module}: {{after}}') """ ) + if additional_code: + script += additional_code + script += textwrap.dedent( + f""" + if unexpected := modules_to_block & sys.modules.keys(): + after = ", ".join(unexpected) + raise AssertionError(f'unexpectedly imported after additional code: {{after}}') + """ + ) + from .script_helper import assert_python_ok assert_python_ok("-S", "-c", script) diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index a41e487f3e4bc52..66e5379d2c6e259 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -149,6 +149,8 @@ def skip_unless_bind_unix_socket(test): """Decorator for tests requiring a functional bind() for unix sockets.""" if not hasattr(socket, 'AF_UNIX'): return unittest.skip('No UNIX Sockets')(test) + if sys.platform == 'cygwin': + return unittest.skip('UNIX sockets hang on Cygwin')(test) global _bind_nix_socket_error if _bind_nix_socket_error is None: from .os_helper import TESTFN, unlink diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index 48fa52bfd5672cb..c7bc6914ac1bc44 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -28,7 +28,7 @@ class TestImportTime(unittest.TestCase): @cpython_only def test_lazy_import(self): import_helper.ensure_lazy_imports( - "_colorize", {"copy", "re", "inspect"} + "_colorize", {"copy", "re", "inspect", "typing"} ) diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 11b2c9545a1b43d..29a29ca0c650974 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -89,6 +89,9 @@ def accept(loc): 'ar_AE': (100, {0: '\u0660', 10: '\u0661\u0660', 99: '\u0669\u0669'}), 'bn_IN': (100, {0: '\u09e6', 10: '\u09e7\u09e6', 99: '\u09ef\u09ef'}), } +if sys.platform == 'cygwin': + count, samples = known_alt_digits['ja_JP'] + known_alt_digits['ja_JP'] = (101, samples) known_era = { 'C': (0, ''), diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index 80ee9e0ba56e754..59a45a2eda07b00 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -11,20 +11,30 @@ import abc import _py_abc from inspect import isabstract +from test.support import warnings_helper def test_factory(abc_ABCMeta, abc_get_cache_token): class TestLegacyAPI(unittest.TestCase): def test_abstractproperty_basics(self): - @abc.abstractproperty - def foo(self): pass + with self.assertWarnsRegex( + DeprecationWarning, + 'abstractproperty', + ): + @abc.abstractproperty + def foo(self): pass + self.assertTrue(foo.__isabstractmethod__) def bar(self): pass self.assertNotHasAttr(bar, "__isabstractmethod__") - class C(metaclass=abc_ABCMeta): - @abc.abstractproperty - def foo(self): return 3 + with self.assertWarnsRegex( + DeprecationWarning, + 'abstractproperty', + ): + class C(metaclass=abc_ABCMeta): + @abc.abstractproperty + def foo(self): return 3 self.assertRaises(TypeError, C) class D(C): @property @@ -33,16 +43,26 @@ def foo(self): return super().foo self.assertFalse(getattr(D.foo, "__isabstractmethod__", False)) def test_abstractclassmethod_basics(self): - @abc.abstractclassmethod - def foo(cls): pass + with self.assertWarnsRegex( + DeprecationWarning, + 'abstractclassmethod', + ): + @abc.abstractclassmethod + def foo(cls): pass + self.assertTrue(foo.__isabstractmethod__) @classmethod def bar(cls): pass self.assertFalse(getattr(bar, "__isabstractmethod__", False)) - class C(metaclass=abc_ABCMeta): - @abc.abstractclassmethod - def foo(cls): return cls.__name__ + with self.assertWarnsRegex( + DeprecationWarning, + 'abstractclassmethod', + ): + class C(metaclass=abc_ABCMeta): + @abc.abstractclassmethod + def foo(cls): return cls.__name__ + self.assertRaises(TypeError, C) class D(C): @classmethod @@ -51,16 +71,26 @@ def foo(cls): return super().foo() self.assertEqual(D().foo(), 'D') def test_abstractstaticmethod_basics(self): - @abc.abstractstaticmethod - def foo(): pass + with self.assertWarnsRegex( + DeprecationWarning, + 'abstractstaticmethod', + ): + @abc.abstractstaticmethod + def foo(): pass + self.assertTrue(foo.__isabstractmethod__) @staticmethod def bar(): pass self.assertFalse(getattr(bar, "__isabstractmethod__", False)) - class C(metaclass=abc_ABCMeta): - @abc.abstractstaticmethod - def foo(): return 3 + with self.assertWarnsRegex( + DeprecationWarning, + 'abstractstaticmethod', + ): + class C(metaclass=abc_ABCMeta): + @abc.abstractstaticmethod + def foo(): return 3 + self.assertRaises(TypeError, C) class D(C): @staticmethod @@ -168,6 +198,7 @@ def method_two(self): msg = r"class C without an implementation for abstract methods 'method_one', 'method_two'" self.assertRaisesRegex(TypeError, msg, C) + @warnings_helper.ignore_warnings(category=DeprecationWarning) def test_abstractmethod_integration(self): for abstractthing in [abc.abstractmethod, abc.abstractproperty, abc.abstractclassmethod, diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index e0c32976fd6f0d1..1dc3f538f4ad8ba 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -80,6 +80,108 @@ def test_skip_invalid_stdout(self): self.assertRegex(mocked_stderr.getvalue(), r'usage:') +class TestLazyImports(unittest.TestCase): + LAZY_IMPORTS = { + "_colorize", + "copy", + "difflib", + "shutil", + "textwrap", + "warnings", + } + def test_module_import(self): + import_helper.ensure_lazy_imports( + "argparse", + self.LAZY_IMPORTS, + ) + + def test_create_parser(self): + # Test imports are still unused after + # creating a parser + create_parser = "argparse.ArgumentParser()" + imported_modules = {"shutil"} + + import_helper.ensure_lazy_imports( + "argparse", + self.LAZY_IMPORTS - imported_modules, + additional_code=create_parser, + ) + + def test_add_subparser(self): + add_subparser = textwrap.dedent( + """ + parser = argparse.ArgumentParser() + parser.add_subparsers(dest='command', required=False) + """ + ) + imported_modules = {"shutil"} + + import_helper.ensure_lazy_imports( + "argparse", + self.LAZY_IMPORTS - imported_modules, + additional_code=add_subparser, + ) + + def test_parse_args(self): + example_parser = textwrap.dedent( + """ + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('-f', '--foo') + parser.add_argument('bar') + parser.parse_args(['BAR']) + parser.parse_args(['BAR', '--foo', 'FOO']) + """ + ) + imported_modules = {"shutil"} + import_helper.ensure_lazy_imports( + "argparse", + self.LAZY_IMPORTS - imported_modules, + additional_code=example_parser + ) + + +class TestArgumentParserCopiable(unittest.TestCase): + def _get_parser(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', type=int, default=42) + parser.add_argument('bar', nargs='?', default='baz') + return parser + + @force_not_colorized + def test_copiable(self): + import copy + parser = self._get_parser() + parser2 = copy.copy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test shallow copy also gets new arguments + parser.add_argument("--extra") + ns3 = parser2.parse_args(["--extra", "bar"]) + self.assertEqual(ns3.extra, "bar") + + @force_not_colorized + def test_deepcopiable(self): + import copy + parser = self._get_parser() + parser2 = copy.deepcopy(parser) + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + # Test deep copy does not get new arguments + parser.add_argument("--extra") + with self.assertRaises(argparse.ArgumentError): + parser2.parse_args(["--extra", "bar"]) + + class TestArgumentParserPickleable(unittest.TestCase): @force_not_colorized @@ -1123,7 +1225,7 @@ def test_invalid_enum_value_raises_error(self): parser.add_argument('--color', choices=self.Color) self.assertRaisesRegex( argparse.ArgumentError, - r"invalid choice: 'yellow' \(choose from red, green, blue\)", + r"invalid choice: 'yellow' \(choose from 'red', 'green', 'blue'\)", parser.parse_args, ['--color', 'yellow'], ) @@ -2392,7 +2494,7 @@ def test_wrong_argument_error_with_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) self.assertIn( - "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from 'bar', 'baz')", excinfo.exception.stderr ) @@ -2402,7 +2504,7 @@ def test_wrong_argument_error_no_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) self.assertIn( - "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + "error: argument foo: invalid choice: 'bazz' (choose from 'bar', 'baz')", excinfo.exception.stderr, ) @@ -2415,7 +2517,7 @@ def test_wrong_argument_subparsers_with_suggestions(self): parser.parse_args(('baz',)) self.assertIn( "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" - " 'bar'? (choose from foo, bar)", + " 'bar'? (choose from 'foo', 'bar')", excinfo.exception.stderr, ) @@ -2427,7 +2529,7 @@ def test_wrong_argument_subparsers_no_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('baz',)) self.assertIn( - "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + "error: argument {foo,bar}: invalid choice: 'baz' (choose from 'foo', 'bar')", excinfo.exception.stderr, ) @@ -2438,7 +2540,7 @@ def test_wrong_argument_with_suggestion_explicit(self): parser.parse_args(('bazz',)) self.assertIn( "error: argument foo: invalid choice: 'bazz', maybe you meant" - " 'baz'? (choose from bar, baz)", + " 'baz'? (choose from 'bar', 'baz')", excinfo.exception.stderr, ) @@ -2458,7 +2560,7 @@ def test_suggestions_choices_int(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('3',)) self.assertIn( - "error: argument foo: invalid choice: '3' (choose from 1, 2)", + "error: argument foo: invalid choice: '3' (choose from '1', '2')", excinfo.exception.stderr, ) @@ -2468,7 +2570,7 @@ def test_suggestions_choices_mixed_types(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('3',)) self.assertIn( - "error: argument foo: invalid choice: '3' (choose from 1, 2)", + "error: argument foo: invalid choice: '3' (choose from '1', '2')", excinfo.exception.stderr, ) @@ -7620,21 +7722,25 @@ def test_backtick_markup_in_description(self): parser = argparse.ArgumentParser( prog='PROG', color=True, - description='Run `python -m myapp` to start.', + description='Run `python myapp` or ``python -m myapp`` to start.', ) prog_extra = self.theme.prog_extra reset = self.theme.reset help_text = parser.format_help() - self.assertIn(f'Run {prog_extra}python -m myapp{reset} to start.', - help_text) + self.assertIn( + f'Run {prog_extra}python myapp{reset} or ' + f'{prog_extra}python -m myapp{reset} to start.', + help_text, + ) + self.assertNotIn("`", help_text) def test_backtick_markup_multiple(self): parser = argparse.ArgumentParser( prog='PROG', color=True, - epilog='Try `app run` or `app test`.', + epilog='Try `app run` or ``app test``.', ) prog_extra = self.theme.prog_extra @@ -7643,17 +7749,19 @@ def test_backtick_markup_multiple(self): help_text = parser.format_help() self.assertIn(f'{prog_extra}app run{reset}', help_text) self.assertIn(f'{prog_extra}app test{reset}', help_text) + self.assertNotIn('`', help_text) def test_backtick_markup_not_applied_when_color_disabled(self): # When color is disabled, backticks are preserved as-is parser = argparse.ArgumentParser( prog='PROG', color=False, - epilog='Example: `python -m myapp`', + epilog='Examples: `python -m myapp` or ``python -m myapp --x``', ) help_text = parser.format_help() self.assertIn('`python -m myapp`', help_text) + self.assertIn('``python -m myapp --x``', help_text) self.assertNotIn('\x1b[', help_text) def test_backtick_markup_with_format_string(self): @@ -7696,6 +7804,39 @@ def test_backtick_markup_special_regex_chars(self): help_text = parser.format_help() self.assertIn(f'{prog_extra}grep "foo.*bar" | sort{reset}', help_text) + def test_backtick_markup_in_argument_help(self): + parser = argparse.ArgumentParser(prog="PROG", color=True) + parser.add_argument("--foo", help="set the `foo` value") + parser.add_argument("--bar", help="set the ``bar`` value") + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f"set the {prog_extra}foo{reset} value", help_text) + self.assertIn(f"set the {prog_extra}bar{reset} value", help_text) + self.assertNotIn("`", help_text) + + def test_backtick_markup_in_argument_help_with_format(self): + parser = argparse.ArgumentParser(prog="PROG", color=True) + parser.add_argument( + "--foo", default="bar", help="set `foo` (default: %(default)s)" + ) + + prog_extra = self.theme.prog_extra + reset = self.theme.reset + + help_text = parser.format_help() + self.assertIn(f"set {prog_extra}foo{reset}", help_text) + + def test_backtick_markup_in_argument_help_color_disabled(self): + parser = argparse.ArgumentParser(prog="PROG", color=False) + parser.add_argument("--foo", help="set the `foo` value") + + help_text = parser.format_help() + self.assertIn("set the `foo` value", help_text) + self.assertNotIn("\x1b[", help_text) + def test_help_with_format_specifiers(self): # GH-142950: format specifiers like %x should work with color=True parser = argparse.ArgumentParser(prog='PROG', color=True) @@ -7762,6 +7903,27 @@ def fake_can_colorize(*, file=None): self.assertIn(output, calls) self.assertNotIn('\x1b[', output.getvalue()) + def test_fake_color_theme_matches_real(self): + from argparse import _colorless_theme + + # Check the attributes match those of the 'real' theme + _colorize_nocolor = _colorize.get_theme(force_no_color=True).argparse + for k in _colorize_nocolor: + self.assertEqual( + getattr(_colorless_theme, k), getattr(_colorize_nocolor, k) + ) + + def test_fake_color_theme_raises(self): + from argparse import _colorless_theme + + # Make sure the _colorless_theme doesn't return empty strings + # for magic methods or private attributes + with self.assertRaises(AttributeError): + _colorless_theme.__unknown_dunder__ + + with self.assertRaises(AttributeError): + _colorless_theme._private_attribute + class TestModule(unittest.TestCase): def test_deprecated__version__(self): diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 54ecc886917dc78..23d9f3c9667d868 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -14,15 +14,10 @@ import operator import struct import sys -import warnings import array from array import _array_reconstructor as array_reconstructor -with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - sizeof_wchar = array.array('u').itemsize - class ArraySubclass(array.array): pass @@ -31,9 +26,7 @@ class ArraySubclassWithKwargs(array.array): def __init__(self, typecode, newarg=None): array.array.__init__(self) -typecodes = ( - 'u', 'w', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', - 'f', 'd', 'q', 'Q', 'F', 'D', 'e', 'Zf', 'Zd') +typecodes = array.typecodes class MiscTest(unittest.TestCase): @@ -135,14 +128,6 @@ def test_typecodes(self): class ArrayReconstructorTest(unittest.TestCase): - def setUp(self): - self.enterContext(warnings.catch_warnings()) - warnings.filterwarnings( - "ignore", - message="The 'u' type code is deprecated and " - "will be removed in Python 3.16", - category=DeprecationWarning) - def test_error(self): self.assertRaises(TypeError, array_reconstructor, "", "b", 0, b"") @@ -210,14 +195,6 @@ def test_numbers(self): [9006104071832581.0, float('inf'), float('-inf'), -0.0]), (['d'], IEEE_754_DOUBLE_BE, '>dddd', [9006104071832581.0, float('inf'), float('-inf'), -0.0]), - (['F'], IEEE_754_FLOAT_COMPLEX_LE, 'FFFF', - [16711938.0j, float('inf'), complex('1-infj'), -0.0]), - (['D'], IEEE_754_DOUBLE_COMPLEX_LE, 'DDDD', - [9006104071832581.0j, float('inf'), complex('1-infj'), -0.0]), (['Zf'], IEEE_754_FLOAT_COMPLEX_LE, 'ZfZfZfZf', @@ -250,12 +227,11 @@ def test_unicode(self): ) for testcase in testcases: mformat_code, encoding = testcase - for c in 'uw': - a = array.array(c, teststr) - b = array_reconstructor( - array.array, c, mformat_code, teststr.encode(encoding)) - self.assertEqual(a, b, - msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) + a = array.array('w', teststr) + b = array_reconstructor( + array.array, 'w', mformat_code, teststr.encode(encoding)) + self.assertEqual(a, b, + msg="{0!r} != {1!r}; testcase={2!r}".format(a, b, testcase)) class BaseTest: @@ -267,14 +243,6 @@ class BaseTest: # outside: An entry that is not in example # minitemsize: the minimum guaranteed itemsize - def setUp(self): - self.enterContext(warnings.catch_warnings()) - warnings.filterwarnings( - "ignore", - message="The 'u' type code is deprecated and " - "will be removed in Python 3.16", - category=DeprecationWarning) - def assertEntryEqual(self, entry1, entry2): self.assertEqual(entry1, entry2) @@ -307,7 +275,7 @@ def test_buffer_info(self): self.assertEqual(bi[1], len(a)) def test_byteswap(self): - if self.typecode in ('u', 'w'): + if self.typecode == 'w': example = '\U00100100' else: example = self.example @@ -1175,7 +1143,7 @@ def test_buffer(self): self.assertEqual(m.tobytes(), expected) self.assertRaises(BufferError, a.frombytes, a.tobytes()) self.assertEqual(m.tobytes(), expected) - if self.typecode in ('u', 'w'): + if self.typecode == 'w': self.assertRaises(BufferError, a.fromunicode, a.tounicode()) self.assertEqual(m.tobytes(), expected) self.assertRaises(BufferError, operator.imul, a, 2) @@ -1231,7 +1199,7 @@ def test_sizeof_without_buffer(self): support.check_sizeof(self, a, basesize) def test_initialize_with_unicode(self): - if self.typecode not in ('u', 'w'): + if self.typecode != 'w': with self.assertRaises(TypeError) as cm: a = array.array(self.typecode, 'foo') self.assertIn("cannot use a str", str(cm.exception)) @@ -1240,7 +1208,6 @@ def test_initialize_with_unicode(self): self.assertIn("cannot use a unicode array", str(cm.exception)) else: a = array.array(self.typecode, "foo") - a = array.array(self.typecode, array.array('u', 'foo')) a = array.array(self.typecode, array.array('w', 'foo')) @support.cpython_only @@ -1266,12 +1233,12 @@ def test_setitem(self): self.assertRaises(TypeError, a.__setitem__, 0, self.example[:2]) class UnicodeTest(StringTest, unittest.TestCase): - typecode = 'u' + typecode = 'w' example = '\x01\u263a\x00\ufeff' smallerexample = '\x01\u263a\x00\ufefe' biggerexample = '\x01\u263a\x01\ufeff' outside = str('\x33') - minitemsize = sizeof_wchar + minitemsize = 4 def test_unicode(self): self.assertRaises(TypeError, array.array, 'b', 'foo') @@ -1293,36 +1260,6 @@ def test_unicode(self): self.assertRaises(TypeError, a.fromunicode) - def test_issue17223(self): - if self.typecode == 'u' and sizeof_wchar == 2: - # PyUnicode_FromUnicode() cannot fail with 16-bit wchar_t - self.skipTest("specific to 32-bit wchar_t") - - # this used to crash - # U+FFFFFFFF is an invalid code point in Unicode 6.0 - invalid_str = b'\xff\xff\xff\xff' - - a = array.array(self.typecode, invalid_str) - self.assertRaises(ValueError, a.tounicode) - self.assertRaises(ValueError, str, a) - - def test_typecode_u_deprecation(self): - with self.assertWarns(DeprecationWarning): - array.array("u") - - def test_empty_string_mem_leak_gh140474(self): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - for _ in range(1000): - a = array.array('u', '') - self.assertEqual(len(a), 0) - self.assertEqual(a.typecode, 'u') - - -class UCS4Test(UnicodeTest): - typecode = 'w' - minitemsize = 4 - class NumberTest(BaseTest): @@ -1645,18 +1582,10 @@ def test_alloc_overflow(self): class ComplexFloatTest(CFPTest, unittest.TestCase): - typecode = 'F' - minitemsize = 8 - -class ComplexDoubleTest(CFPTest, unittest.TestCase): - typecode = 'D' - minitemsize = 16 - -class ComplexZfFloatTest(CFPTest, unittest.TestCase): typecode = 'Zf' minitemsize = 8 -class ComplexZdDoubleTest(CFPTest, unittest.TestCase): +class ComplexDoubleTest(CFPTest, unittest.TestCase): typecode = 'Zd' minitemsize = 16 diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index af8b334da03e702..fd3b33bab7f8336 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -523,6 +523,12 @@ def test_classattrs(self): self.assertIs(ast.Constant(None).value, None) self.assertIs(ast.Constant(...).value, ...) + with self.assertWarns(DeprecationWarning): + ast.Tuple().dims + + with self.assertWarns(DeprecationWarning): + ast.Tuple().dims = 3 + def test_constant_subclasses(self): class N(ast.Constant): def __init__(self, *args, **kwargs): @@ -619,7 +625,7 @@ def test_invalid_identifier(self): ast.fix_missing_locations(m) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") - self.assertIn("identifier must be of type str", str(cm.exception)) + self.assertIn("expecting a string object", str(cm.exception)) def test_invalid_constant(self): for invalid_constant in int, (1, 2, int), frozenset((1, 2, int)): @@ -1075,6 +1081,30 @@ def test_none_checks(self) -> None: for node, attr, source in tests: self.assert_none_check(node, attr, source) + def test_required_field_messages(self): + binop = ast.BinOp( + left=ast.Constant(value=2), + right=ast.Constant(value=2), + op=ast.Add(), + ) + expr_without_position = ast.Expression(body=binop) + expr_with_wrong_body = ast.Expression(body=[binop]) + + with self.assertRaisesRegex(TypeError, "required field") as cm: + compile(expr_without_position, "", "eval") + with self.assertRaisesRegex( + TypeError, + "field 'body' was expecting node of type 'expr', got 'list'", + ): + compile(expr_with_wrong_body, "", "eval") + + constant = ast.parse("u'test'", mode="eval") + constant.body.kind = 0xFF + with self.assertRaisesRegex( + TypeError, "field 'kind' was expecting a string or bytes object" + ): + compile(constant, "", "eval") + def test_repr(self) -> None: snapshots = AST_REPR_DATA_FILE.read_text().split("\n") for test, snapshot in zip(ast_repr_get_test_cases(), snapshots, strict=True): @@ -1100,6 +1130,38 @@ def test_tstring(self): self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) + def test_deprecated(self): + with self.assertWarns(DeprecationWarning): + ast.slice + + with self.assertWarns(DeprecationWarning): + ast.Index + + with self.assertWarns(DeprecationWarning): + ast.ExtSlice + + with self.assertWarns(DeprecationWarning): + ast.Suite + + with self.assertWarns(DeprecationWarning): + ast.AugLoad + + with self.assertWarns(DeprecationWarning): + ast.AugStore + + with self.assertWarns(DeprecationWarning): + ast.Param + + namespace = {} + exec("from ast import *", namespace) + self.assertNotIn("slice", namespace) + self.assertNotIn("Index", namespace) + self.assertNotIn("ExtSlice", namespace) + self.assertNotIn("Suite", namespace) + self.assertNotIn("AugLoad", namespace) + self.assertNotIn("AugStore", namespace) + self.assertNotIn("Param", namespace) + def test_filter_syntax_warnings_by_module(self): filename = support.findfile('test_import/data/syntax_warnings.py') with open(filename, 'rb') as f: @@ -1139,8 +1201,9 @@ def iter_ast_classes(): def do(cls): if cls.__module__ != 'ast': return - if cls is ast.Index: - return + with warnings.catch_warnings(action="ignore", category=DeprecationWarning): + if cls is ast.Index: + return # Don't attempt to create instances of abstract AST nodes if _ast._is_abstract(cls): return diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py index d874ed00bd7e7ad..0e149dadd7f1219 100644 --- a/Lib/test/test_asyncio/test_free_threading.py +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -165,6 +165,45 @@ async def main(): loop.set_task_factory(self.factory) r.run(main()) + def test_all_tasks_from_other_thread_includes_eager_tasks(self): + # gh-152020: all_tasks() called from another thread used to drop + # eager-started tasks on free-threaded builds. + loop = asyncio.new_event_loop() + + async def wait_forever(): + await asyncio.Event().wait() + + def eager_factory(loop, coro, **kwargs): + return self.factory(loop, coro, eager_start=True, **kwargs) + + async def setup(): + loop.set_task_factory(eager_factory) + eager = loop.create_task(wait_forever(), name="EAGER") + loop.set_task_factory(None) + normal = loop.create_task(wait_forever(), name="NORMAL") + return eager, normal + + async def teardown(): + tasks = [t for t in asyncio.all_tasks() + if t is not asyncio.current_task()] + for t in tasks: + t.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + thread = threading.Thread(target=loop.run_forever) + thread.start() + try: + held = asyncio.run_coroutine_threadsafe(setup(), loop).result() + names = {t.get_name() for t in asyncio.all_tasks(loop)} + self.assertIn("NORMAL", names) + self.assertIn("EAGER", names) + del held + finally: + asyncio.run_coroutine_threadsafe(teardown(), loop).result() + loop.call_soon_threadsafe(loop.stop) + thread.join() + loop.close() + class TestPyFreeThreading(TestFreeThreading, TestCase): diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 2f22fbccba42bc8..cffb2b30ad9cff6 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -1,6 +1,7 @@ import asyncio import io import unittest +from unittest import mock # To prevent a warning "test altered the execution environment" @@ -345,6 +346,139 @@ async def main(): self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) + async def test_capture_call_graph_positive_limit(self): + captured = None + + async def c3(): + nonlocal captured + captured = asyncio.capture_call_graph(limit=2) + + async def c2(): + await c3() + + async def c1(): + await c2() + + await c1() + self.assertEqual(len(captured.call_stack), 2) + + async def test_capture_call_graph_negative_limit(self): + captured = None + + async def c3(): + nonlocal captured + captured = asyncio.capture_call_graph(limit=-2) + + async def c2(): + await c3() + + async def c1(): + await c2() + + await c1() + self.assertEqual(len(captured.call_stack), 2) + + async def test_capture_call_graph_zero_limit(self): + captured = None + + async def inner(): + nonlocal captured + captured = asyncio.capture_call_graph(limit=0) + + await inner() + self.assertEqual(captured.call_stack, ()) + + def test_capture_call_graph_outside_loop(self): + with self.assertRaises(RuntimeError): + asyncio.capture_call_graph() + + def test_capture_call_graph_non_future(self): + with self.assertRaises(TypeError): + asyncio.capture_call_graph("not a future") + + async def test_capture_call_graph_no_current_task(self): + results = [] + + def cb(): + results.append(asyncio.capture_call_graph()) + results.append(asyncio.format_call_graph()) + + loop = asyncio.get_running_loop() + loop.call_soon(cb) + await asyncio.sleep(0) + + self.assertEqual(results, [None, ""]) + + async def test_capture_call_graph_current_task_not_future(self): + sentinel = object() + with mock.patch('asyncio.tasks.current_task', return_value=sentinel): + with self.assertRaises(TypeError): + asyncio.capture_call_graph(sentinel) + + async def test_build_graph_for_future_positive_limit(self): + fut = asyncio.Future() + captured = None + + async def deep(): + await fut + + async def mid(): + await deep() + + async def runner(): + await mid() + + async def main(): + nonlocal captured + async with asyncio.TaskGroup() as g: + t = g.create_task(runner(), name='runner') + for _ in range(5): + await asyncio.sleep(0) + captured = asyncio.capture_call_graph(t, limit=2) + fut.set_result(None) + + await main() + self.assertEqual(len(captured.call_stack), 2) + + async def test_build_graph_for_future_negative_limit(self): + fut = asyncio.Future() + captured = None + + async def deep(): + await fut + + async def mid(): + await deep() + + async def runner(): + await mid() + + async def main(): + nonlocal captured + async with asyncio.TaskGroup() as g: + t = g.create_task(runner(), name='runner') + for _ in range(5): + await asyncio.sleep(0) + captured = asyncio.capture_call_graph(t, limit=-2) + fut.set_result(None) + + await main() + self.assertEqual(len(captured.call_stack), 2) + + async def test_format_call_graph_regular_generator(self): + output = [] + + def gen(): + output.append(asyncio.format_call_graph()) + yield + + async def main(): + for _ in gen(): + pass + + await main() + self.assertRegex(output[0], r'in generator [\w.<>]+\.gen\(\)') + @unittest.skipIf( not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index a0c8434c9457d2c..95a9f3a9a7cf716 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -124,11 +124,6 @@ def foo(): yield self.assertFalse(asyncio.iscoroutine(foo())) - def test_iscoroutinefunction(self): - async def foo(): pass - with self.assertWarns(DeprecationWarning): - self.assertTrue(asyncio.iscoroutinefunction(foo)) - def test_async_def_coroutines(self): async def bar(): return 'spam' diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index dcd963b3355ef86..7afd7de3bb936e6 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -228,6 +228,61 @@ def test_sock_sendfile_zero_size(self): self.assertEqual(ret, 0) self.assertEqual(self.file.tell(), 0) + def check_sock_sendfile_offset(self, data, offset, force_fallback=False): + sock, proto = self.prepare_socksendfile() + with tempfile.TemporaryFile() as f: + f.write(data) + f.flush() + self.assertEqual(f.tell(), len(data)) + + if force_fallback: + async def _sock_sendfile_fail(sock, file, offset, count): + raise asyncio.exceptions.SendfileNotAvailableError() + with support.swap_attr(self.loop, '_sock_sendfile_native', _sock_sendfile_fail): + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + else: + ret = self.run_loop(self.loop.sock_sendfile(sock, f, offset, None)) + + self.assertEqual(f.tell(), len(data)) + + sock.close() + self.run_loop(proto.wait_closed()) + + self.assertEqual(ret, len(data) - offset) + + + def test_sock_sendfile_offset(self): + data = b'abcdef' + for offset in (0, len(data) // 2, len(data)): + for force_fallback in (False, True): + with self.subTest(offset=offset, force_fallback=force_fallback): + self.check_sock_sendfile_offset(data, offset, force_fallback) + + def check_sendfile_offset(self, offset, fallback): + srv_proto, cli_proto = self.prepare_sendfile() + self.file.seek(123) + coro = self.loop.sendfile(cli_proto.transport, self.file, offset, fallback=fallback) + try: + ret = self.run_loop(coro) + except asyncio.SendfileNotAvailableError: + if fallback: + raise + cli_proto.transport.close() + self.run_loop(srv_proto.done) + return + cli_proto.transport.close() + self.run_loop(srv_proto.done) + self.assertEqual(ret, len(self.DATA) - offset) + self.assertEqual(srv_proto.nbytes, len(self.DATA) - offset) + self.assertEqual(srv_proto.data, self.DATA[offset:]) + self.assertEqual(self.file.tell(), len(self.DATA)) + + def test_sendfile_offset(self): + for offset in (0, len(self.DATA) // 2, len(self.DATA)): + for fallback in (False, True): + with self.subTest(offset=offset, fallback=fallback): + self.check_sendfile_offset(offset, fallback) + def test_sock_sendfile_mix_with_regular_send(self): buf = b"mix_regular_send" * (4 * 1024) # 64 KiB sock, proto = self.prepare_socksendfile() diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index ca15fc3bdd42dd7..784c784d261dc66 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -1544,6 +1544,9 @@ async def client(addr): # This triggers bug gh-115514, also tested using mocks in # test.test_asyncio.test_selector_events.SelectorSocketTransportTests.test_write_buffer_after_close socket_transport = writer.transport._ssl_protocol._transport + # connection_lost may have already cleared _transport. + if socket_transport is None: + return class SocketWrapper: def __init__(self, sock) -> None: diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index c08eb7cf2615680..4ac6b23b7120fcc 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -1,4 +1,5 @@ import os +import shlex import signal import sys import textwrap @@ -770,9 +771,7 @@ async def check_stdout_output(self, coro, output): def test_create_subprocess_env_shell(self) -> None: async def main() -> None: - executable = sys.executable - if sys.platform == "win32": - executable = f'"{executable}"' + executable = f'"{sys.executable}"' if sys.platform == "win32" else shlex.quote(sys.executable) cmd = f'''{executable} -c "import os, sys; sys.stdout.write(os.getenv('FOO'))"''' env = os.environ.copy() env["FOO"] = "bar" diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index dc179acd86e8a64..56b1494c8363ca6 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -11,6 +11,7 @@ import traceback import types import unittest +import inspect from unittest import mock from types import GenericAlias @@ -20,7 +21,6 @@ from test.test_asyncio import utils as test_utils from test import support from test.support.script_helper import assert_python_ok -from test.support.warnings_helper import ignore_warnings def tearDownModule(): @@ -1940,30 +1940,11 @@ async def notmutch(): self.assertFalse(task.cancelled()) self.assertIs(task.exception(), base_exc) - @ignore_warnings(category=DeprecationWarning) - def test_iscoroutinefunction(self): - def fn(): - pass - - self.assertFalse(asyncio.iscoroutinefunction(fn)) - - def fn1(): - yield - self.assertFalse(asyncio.iscoroutinefunction(fn1)) - - async def fn2(): - pass - self.assertTrue(asyncio.iscoroutinefunction(fn2)) - - self.assertFalse(asyncio.iscoroutinefunction(mock.Mock())) - self.assertTrue(asyncio.iscoroutinefunction(mock.AsyncMock())) - - @ignore_warnings(category=DeprecationWarning) def test_coroutine_non_gen_function(self): async def func(): return 'test' - self.assertTrue(asyncio.iscoroutinefunction(func)) + self.assertTrue(inspect.iscoroutinefunction(func)) coro = func() self.assertTrue(asyncio.iscoroutine(coro)) diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index dbb572e2e1536b3..1781f1f67753f3a 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -98,6 +98,26 @@ def get_write_buffer_size(self): self.assertTrue(transport._protocol_paused) self.assertEqual(transport.get_write_buffer_limits(), (128, 256)) + def test_flowcontrol_mixin_compute_write_limits(self): + + class MyTransport(transports._FlowControlMixin, + transports.Transport): + + def get_write_buffer_size(self): + return 0 + + loop = mock.Mock() + transport = MyTransport(loop=loop) + + self.assertEqual(transport.get_write_buffer_limits(), + (16 * 1024, 64 * 1024)) + + transport.set_write_buffer_limits(low=100) + self.assertEqual(transport.get_write_buffer_limits(), (100, 400)) + + transport.set_write_buffer_limits(high=200) + self.assertEqual(transport.get_write_buffer_limits(), (50, 200)) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index d2b3de3b9a4cb61..118cb866997c51c 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -1333,5 +1333,45 @@ async def child_main(): self.assertEqual(result.value, 0) + +@unittest.skipUnless( + unix_events.can_use_pidfd(), + "operating system does not support pidfd", +) +class PidfdChildWatcherTests(test_utils.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.new_event_loop() + self.set_event_loop(self.loop) + + def test_pidfd_closed_when_waitpid_raises(self): + # _do_wait() must close the pidfd even when waitpid() + # fails with something other than ChildProcessError, otherwise the + # pidfd is leaked + self.loop.set_exception_handler(lambda loop, context: None) + + async def coro(): + before = os_helper.fd_count() + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', 'import sys; sys.stdin.read()', + stdin=asyncio.subprocess.PIPE + ) + + with mock.patch.object(os, 'waitpid', + side_effect=OSError('unexpected')) as m: + proc.stdin.close() + while not m.called: + await asyncio.sleep(0) + + os.waitpid(proc.pid, 0) + proc._transport._process_exited(0) + await proc.wait() + + self.assertEqual(os_helper.fd_count(), before) + + self.loop.run_until_complete(coro()) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index f9ee2f4f68150a1..509697613475953 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -77,6 +77,30 @@ def test_pipe_handle(self): else: raise RuntimeError('expected ERROR_INVALID_HANDLE') + def test_pipe_handle_close_after_external_close(self): + # gh-149388: PipeHandle.close() must clear ``_handle`` before calling + # CloseHandle so that if CloseHandle raises on a stale handle the + # PipeHandle is still marked closed and __del__ / subsequent close() + # calls are silent no-ops. + h1, h2 = windows_utils.pipe(overlapped=(False, False)) + try: + p = windows_utils.PipeHandle(h1) + # Simulate an external close of the underlying handle (e.g. + # a finalizer race or a concurrent close on the same object). + _winapi.CloseHandle(p.handle) + # First close() still propagates the OSError from CloseHandle, + # but must clear ``_handle`` first. + with self.assertRaises(OSError): + p.close() + self.assertIsNone(p.handle) + # Second close() is a no-op. + p.close() + # __del__ through GC is also a silent no-op — no unraisable. + del p + support.gc_collect() + finally: + _winapi.CloseHandle(h2) + class PopenTests(unittest.TestCase): diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 6d0879ad65d53c6..7f53379a952dfc1 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -10,7 +10,6 @@ from test import support from test.support import bigmemtest, _1G, _2G, _4G, import_helper -_testcapi = import_helper.import_module('_testcapi') import unittest import operator @@ -1264,6 +1263,7 @@ class ImmortalityTest(unittest.TestCase): def test_stickiness(self, size): """Check that immortality is "sticky", so that once an object is immortal it remains so.""" + _testcapi = import_helper.import_module('_testcapi') if size < _2G: # Not enough memory to cause immortality on overflow return diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 21a3d0b3422ad2f..3213a4751273431 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -24,7 +24,6 @@ import sys, array, io, os from decimal import Decimal from fractions import Fraction -from test.support import warnings_helper try: from _testbuffer import * @@ -67,7 +66,7 @@ 'h':0, 'H':0, 'i':0, 'I':0, 'l':0, 'L':0, 'n':0, 'N':0, 'e':0, 'f':0, 'd':0, 'P':0, - 'F':0, 'D':0, 'Zf':0, 'Zd':0, + 'Zf':0, 'Zd':0, } # NumPy does not have 'n' or 'N': @@ -94,8 +93,6 @@ 'q':(-(1<<63), 1<<63), 'Q':(0, 1<<64), 'e':(-65519, 65520), 'f':(-(1<<63), 1<<63), 'd':(-(1<<1023), 1<<1023), - 'F':(-(1<<63), 1<<63), - 'D':(-(1<<1023), 1<<1023), 'Zf':(-(1<<63), 1<<63), 'Zd':(-(1<<1023), 1<<1023), } @@ -112,9 +109,9 @@ def native_type_range(fmt): lh = (-(1<<63), 1<<63) elif fmt == 'd': lh = (-(1<<1023), 1<<1023) - elif fmt in ('F', 'Zf'): + elif fmt == 'Zf': lh = (-(1<<63), 1<<63) - elif fmt in ('D', 'Zd'): + elif fmt == 'Zd': lh = (-(1<<1023), 1<<1023) else: for exp in (128, 127, 64, 63, 32, 31, 16, 15, 8, 7): @@ -145,7 +142,7 @@ def native_type_range(fmt): # Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: - if not k in "bBhHiIlLfd": + if k not in list("bBhHiIlLefd") + ['Zf', 'Zd']: del ARRAY[k] BYTEFMT = NATIVE.copy() @@ -184,7 +181,7 @@ def randrange_fmt(mode, char, obj): if char in 'efd': x = struct.pack(char, x) x = struct.unpack(char, x)[0] - if char in ('F', 'D', 'Zf', 'Zd'): + if char in ('Zf', 'Zd'): y = randrange(*fmtdict[mode][char]) x = complex(x, y) x = struct.pack(char, x) @@ -3055,7 +3052,7 @@ def test_memoryview_assign(self): continue m2 = m1.cast(fmt) lo, hi = _range - if fmt in ("d", "f", "D", "F", "Zd", "Zf"): + if fmt in ("d", "f", "Zd", "Zf"): lo, hi = -2**1024, 2**1024 if fmt != 'P': # PyLong_AsVoidPtr() accepts negative numbers self.assertRaises(ValueError, m2.__setitem__, 0, lo-1) @@ -3263,15 +3260,6 @@ class BEPoint(ctypes.BigEndianStructure): self.assertNotEqual(point, a) self.assertRaises(NotImplementedError, a.tolist) - @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') - def test_memoryview_compare_special_cases_deprecated_u_type_code(self): - - # Depends on issue #15625: the struct module does not understand 'u'. - a = array.array('u', 'xyz') - v = memoryview(a) - self.assertNotEqual(a, v) - self.assertNotEqual(v, a) - def test_memoryview_compare_ndim_zero(self): nd1 = ndarray(1729, shape=[], format='@L') @@ -4497,8 +4485,10 @@ def test_bytearray_alignment(self): def test_array_alignment(self): # gh-140557: pointer alignment of buffers including empty allocation # should match the maximum array alignment. - align = max(struct.calcsize(fmt) for fmt in ARRAY) - cases = [array.array(fmt) for fmt in ARRAY] + formats = [fmt for fmt in ARRAY + if struct.calcsize(fmt) <= struct.calcsize('P')] + align = max(struct.calcsize(fmt) for fmt in formats) + cases = [array.array(fmt) for fmt in formats] # Empty arrays self.assertEqual( [_testcapi.buffer_pointer_as_int(case) % align for case in cases], diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 81967fb8a837404..1d2c105ac047e18 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -39,7 +39,7 @@ from test.support.script_helper import assert_python_ok from test.support.testcase import ComplexesAreIdenticalMixin from test.support.warnings_helper import check_warnings -from test.support import requires_IEEE_754 +from test.support import requires_IEEE_754, skip_if_double_rounding from unittest.mock import MagicMock, patch try: import pty, signal @@ -47,11 +47,6 @@ pty = signal = None -# Detect evidence of double-rounding: sum() does not always -# get improved accuracy on machines that suffer from double rounding. -x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer -HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) - # used as proof of globals being used A_GLOBAL_VALUE = 123 A_SENTINEL = sentinel("A_SENTINEL") @@ -268,7 +263,10 @@ def f_list(): def f_set(): return set(2*x for x in [1,2,3]) - funcs = [f_all, f_any, f_tuple, f_list, f_set] + def f_frozenset(): + return frozenset(2*x for x in [1,2,3]) + + funcs = [f_all, f_any, f_tuple, f_list, f_set, f_frozenset] for f in funcs: # check that generator code object is not duplicated @@ -278,35 +276,37 @@ def f_set(): # check the overriding the builtins works - global all, any, tuple, list, set - saved = all, any, tuple, list, set + global all, any, tuple, list, set, frozenset + saved = all, any, tuple, list, set, frozenset try: all = lambda x : "all" any = lambda x : "any" tuple = lambda x : "tuple" list = lambda x : "list" set = lambda x : "set" + frozenset = lambda x : "frozenset" overridden_outputs = [f() for f in funcs] finally: - all, any, tuple, list, set = saved + all, any, tuple, list, set, frozenset = saved - self.assertEqual(overridden_outputs, ['all', 'any', 'tuple', 'list', 'set']) + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple', 'list', 'set', 'frozenset']) # Now repeat, overriding the builtins module as well - saved = all, any, tuple, list, set + saved = all, any, tuple, list, set, frozenset try: builtins.all = all = lambda x : "all" builtins.any = any = lambda x : "any" builtins.tuple = tuple = lambda x : "tuple" builtins.list = list = lambda x : "list" builtins.set = set = lambda x : "set" + builtins.frozenset = frozenset = lambda x : "frozenset" overridden_outputs = [f() for f in funcs] finally: - all, any, tuple, list, set = saved - builtins.all, builtins.any, builtins.tuple, builtins.list, builtins.set = saved + all, any, tuple, list, set, frozenset = saved + builtins.all, builtins.any, builtins.tuple, builtins.list, builtins.set, builtins.frozenset = saved - self.assertEqual(overridden_outputs, ['all', 'any', 'tuple', 'list', 'set']) + self.assertEqual(overridden_outputs, ['all', 'any', 'tuple', 'list', 'set', 'frozenset']) def test_builtin_call_async_genexpr_no_crash(self): async def f_all(): @@ -1956,16 +1956,33 @@ def test_sentinel(self): with self.assertRaises(TypeError): class SubSentinel(sentinel): pass + + def test_sentinel_attributes(self): + missing = sentinel("MISSING") with self.assertRaises(TypeError): sentinel.attribute = "value" with self.assertRaises(AttributeError): - missing.__name__ = "CHANGED" + missing.attribute = "value" with self.assertRaises(AttributeError): - missing.__module__ = "changed" + missing.__name__ = "CHANGED" + missing.__module__ = "changed" + self.assertEqual(missing.__module__, "changed") with self.assertRaises(AttributeError): del missing.__name__ + del missing.__module__ with self.assertRaises(AttributeError): - del missing.__module__ + missing.__module__ + + def test_sentinel_repr(self): + with_repr = sentinel("WITH_REPR", repr="custom") + without_repr = sentinel("WITHOUT_REPR", repr=None) + self.assertEqual(repr(with_repr), "custom") + self.assertEqual(repr(without_repr), "WITHOUT_REPR") + self.assertEqual(str(with_repr), "custom") + self.assertEqual(str(without_repr), "WITHOUT_REPR") + + with self.assertRaisesRegex(TypeError, "repr.*str or None"): + sentinel("BAD_REPR", repr=42) def test_sentinel_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -2213,8 +2230,7 @@ def __getitem__(self, index): complex(2, -0.0)) @requires_IEEE_754 - @unittest.skipIf(HAVE_DOUBLE_ROUNDING, - "sum accuracy not guaranteed on machines with double rounding") + @skip_if_double_rounding @support.cpython_only # Other implementations may choose a different algorithm def test_sum_accuracy(self): self.assertEqual(sum([0.1] * 10), 1.0) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index b1cdbe04765ed0d..e211c3d15a4ed20 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -645,6 +645,32 @@ def test_join(self): with self.assertRaises(TypeError): dot_join([memoryview(b"ab"), "cd", b"ef"]) + def test_join_concurrent_buffer_mutation(self): + # __buffer__() can release the GIL, letting another thread concurrently + # mutate the joined sequence (simulated here by mutating in __buffer__). + # See: https://github.com/python/cpython/issues/151295 + def make_seq(mutate): + # Item is only referenced from the list slot, so mutate() frees it. + class Item: + def __buffer__(self, flags): + mutate(seq) + return memoryview(b'x') + seq = [b'a', Item(), b'c'] + return seq + + for sep in (self.type2test(b''), self.type2test(b'::')): + with self.subTest(sep=sep): + # Changing the list length is reported as a RuntimeError. + seq = make_seq(lambda seq: seq.clear()) + self.assertRaises(RuntimeError, sep.join, seq) + + # The list length is unchanged, so the size-change recheck + # cannot fire: only keeping the item alive avoids the crash. + def replace(seq): + seq[1] = b'z' + seq = make_seq(replace) + self.assertEqual(sep.join(seq), sep.join([b'a', b'x', b'c'])) + def test_count(self): b = self.type2test(b'mississippi') i = 105 @@ -2700,10 +2726,6 @@ def iconcat(b, a): # MODIFIES! b.wait() a += c - def irepeat(b, a): # MODIFIES! - b.wait() - a *= 2 - def subscript(b, a): b.wait() try: assert a[0] != 0xdd @@ -2837,9 +2859,10 @@ def check(funcs, a=None, *args): check([clear] + [repeat] * 10) check([clear] + [iconcat] * 10) - check([clear] + [irepeat] * 10) check([clear] + [ass_subscript] * 10) check([clear] + [repr_] * 10) + # gh-148605: Do not test "a *= 2" since it allocates up to 4 GiB using + # 10 threads # value errors diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index d8e3b671ec229f9..64293d757331d75 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1032,6 +1032,21 @@ def test_failure(self): # Previously, a second call could crash due to internal inconsistency self.assertRaises(Exception, bzd.decompress, self.BAD_DATA * 30) + def test_decompress_after_data_error(self): + data = bytes.fromhex( + "425a6839314159265359000000000000007fffff000000000000000000000000" + "00000000000000000000000000000000000000e0370000000000000000000000" + "000000000000000000000000000000000000000000000000000083f3" + ) + bzd = BZ2Decompressor() + with self.assertRaisesRegex(OSError, "Invalid data stream"): + bzd.decompress(data) + # Previously, a second call could crash due to internal inconsistency + self.assertFalse(bzd.needs_input) + self.assertFalse(bzd.eof) + with self.assertRaisesRegex(ValueError, "previous error"): + bzd.decompress(b'\x00' * 18) + @support.refcount_test def test_refleaks_in___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index 340bec3c71b68f8..52323a0ec0ac2bb 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -47,9 +47,8 @@ # FS encoding is UTF-8 on macOS EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" elif sys.platform == "cygwin": - # Cygwin defaults to using C.UTF-8 - # TODO: Work out a robust dynamic test for this that doesn't rely on - # CPython's own locale handling machinery + DEFAULT_LOCALE_IS_C = False + DEFAULT_ENCODING = "utf-8" EXPECT_COERCION_IN_DEFAULT_LOCALE = False elif sys.platform == "vxworks": # VxWorks defaults to using UTF-8 for all system interfaces diff --git a/Lib/test/test_frame_pointer_unwind.py b/Lib/test/test_c_stack_unwind.py similarity index 52% rename from Lib/test/test_frame_pointer_unwind.py rename to Lib/test/test_c_stack_unwind.py index 2f9ce2bf049f588..91bf44e463473de 100644 --- a/Lib/test/test_frame_pointer_unwind.py +++ b/Lib/test/test_c_stack_unwind.py @@ -1,3 +1,12 @@ +"""Test in-process C stack unwinders against Python and JIT frames. + +The tests build a recursive Python call stack, ask each _testinternalcapi +unwinder for return addresses, and classify those addresses as Python, JIT, or +other frames. The backends include CPython's manual stack-chain unwinder and +GNU backtrace(), so this module is about in-process C stack unwinding rather +than a single unwind mechanism. GDB integration tests live in test_gdb. +""" + import json import os import platform @@ -17,7 +26,20 @@ raise unittest.SkipTest("test requires subprocess support") -def _frame_pointers_expected(machine): +STACK_DEPTH = 10 + + +def _manual_unwind_expected(machine): + _Py_WITH_FRAME_POINTERS = getattr( + _testinternalcapi, + "_Py_WITH_FRAME_POINTERS", + -1, + ) + if _Py_WITH_FRAME_POINTERS > 0: + return True + if _Py_WITH_FRAME_POINTERS == 0: + return False + cflags = " ".join( value for value in ( sysconfig.get_config_var("PY_CORE_CFLAGS"), @@ -43,6 +65,12 @@ def _frame_pointers_expected(machine): if sys.maxsize < 2**32: return None return True + if machine == "ppc64le": + # The power ABI specification requires that compilers maintain a + # back chain by default, so unwinding already works without a + # dedicated frame pointer. + # https://openpowerfoundation.org/specifications/64bitelfabi/ + return True if machine == "x86_64": final_opt = "" for opt in cflags.split(): @@ -70,7 +98,22 @@ def _frame_pointers_expected(machine): return None -def _build_stack_and_unwind(): +def _is_arm32_build(): + if sys.maxsize >= 2**32: + return False + + abi = " ".join( + value for value in ( + sysconfig.get_config_var("MULTIARCH"), + sysconfig.get_config_var("HOST_GNU_TYPE"), + sysconfig.get_config_var("SOABI"), + ) + if value + ).lower() + return "arm" in abi + + +def _build_stack_and_unwind(unwinder): import operator def build_stack(n, unwinder, warming_up_caller=False): @@ -89,7 +132,7 @@ def build_stack(n, unwinder, warming_up_caller=False): result = operator.call(build_stack, n - 1, unwinder, warming_up) return result - stack = build_stack(10, _testinternalcapi.manual_frame_pointer_unwind) + stack = build_stack(STACK_DEPTH, unwinder) return stack @@ -112,8 +155,7 @@ def _classify_stack(stack, jit_enabled): return annotated, python_frames, jit_frames, other_frames -def _annotate_unwind(): - stack = _build_stack_and_unwind() +def _summarize_unwind(stack, unwinder_name): jit_enabled = hasattr(sys, "_jit") and sys._jit.is_enabled() jit_backend = _testinternalcapi.get_jit_backend() ranges = _testinternalcapi.get_jit_code_ranges() if jit_enabled else [] @@ -126,19 +168,44 @@ def _annotate_unwind(): ) for idx, addr, tag in annotated: print(f"#{idx:02d} {addr:#x} -> {tag}") - return json.dumps({ + return { "length": len(stack), "python_frames": python_frames, "jit_frames": jit_frames, "other_frames": other_frames, "jit_backend": jit_backend, + "unwinder": unwinder_name, + } + + +def _annotate_unwind(unwinder_name="manual_frame_pointer_unwind"): + unwinder = getattr(_testinternalcapi, unwinder_name) + stack = _build_stack_and_unwind(unwinder) + return json.dumps(_summarize_unwind(stack, unwinder_name)) + + +def _annotate_unwind_after_executor_free(unwinder_name="gnu_backtrace_unwind"): + # The first unwind runs at the bottom of _build_stack_and_unwind(), while + # the recursive helper may be executing in JIT code. After it returns, this + # helper is back in normal test code; clearing executor caches should remove + # the old JIT ranges, so the second unwind must not report stale JIT frames. + live = json.loads(_annotate_unwind(unwinder_name)) + + sys._clear_internal_caches() + _testinternalcapi.clear_executor_deletion_list() + + unwinder = getattr(_testinternalcapi, unwinder_name) + after_free = _summarize_unwind(unwinder(), unwinder_name) + return json.dumps({ + "live": live, + "after_free": after_free, }) -def _manual_unwind_length(**env): +def _run_unwind_helper(helper_name, unwinder_name, **env): code = ( - "from test.test_frame_pointer_unwind import _annotate_unwind; " - "print(_annotate_unwind());" + f"from test.test_c_stack_unwind import {helper_name}; " + f"print({helper_name}({unwinder_name!r}));" ) run_env = os.environ.copy() run_env.update(env) @@ -166,17 +233,28 @@ def _manual_unwind_length(**env): ) from exc +def _unwind_result(unwinder_name, **env): + return _run_unwind_helper("_annotate_unwind", unwinder_name, **env) + + +def _unwind_after_executor_free_result(unwinder_name, **env): + return _run_unwind_helper( + "_annotate_unwind_after_executor_free", unwinder_name, **env) + + @support.requires_gil_enabled("test requires the GIL enabled") @unittest.skipIf(support.is_wasi, "test not supported on WASI") -class FramePointerUnwindTests(unittest.TestCase): +class ManualStackUnwindTests(unittest.TestCase): def setUp(self): super().setUp() machine = platform.machine().lower() - expected = _frame_pointers_expected(machine) + expected = _manual_unwind_expected(machine) if expected is None: - self.skipTest(f"unsupported architecture for frame pointer check: {machine}") + self.skipTest( + f"unsupported architecture for manual stack unwind check: {machine}" + ) if expected == "crash": self.skipTest(f"test does crash on {machine}") @@ -184,12 +262,14 @@ def setUp(self): _testinternalcapi.manual_frame_pointer_unwind() except RuntimeError as exc: if "not supported" in str(exc): - self.skipTest("manual frame pointer unwinding not supported on this platform") + self.skipTest( + "manual stack unwinding not supported on this platform" + ) raise self.machine = machine - self.frame_pointers_expected = expected + self.manual_unwind_expected = expected - def test_manual_unwind_respects_frame_pointers(self): + def test_manual_unwind_finds_expected_frames(self): jit_available = hasattr(sys, "_jit") and sys._jit.is_available() envs = [({"PYTHON_JIT": "0"}, False)] if jit_available: @@ -197,14 +277,14 @@ def test_manual_unwind_respects_frame_pointers(self): for env, using_jit in envs: with self.subTest(env=env): - result = _manual_unwind_length(**env) + result = _unwind_result("manual_frame_pointer_unwind", **env) jit_frames = result["jit_frames"] python_frames = result.get("python_frames", 0) jit_backend = result.get("jit_backend") - if self.frame_pointers_expected: - self.assertGreater( + if self.manual_unwind_expected: + self.assertGreaterEqual( python_frames, - 0, + STACK_DEPTH, f"expected to find Python frames on {self.machine} with env {env}", ) if using_jit: @@ -240,5 +320,88 @@ def test_manual_unwind_respects_frame_pointers(self): ) +@support.requires_gil_enabled("test requires the GIL enabled") +@unittest.skipIf(support.is_wasi, "test not supported on WASI") +@unittest.skipUnless(sys.platform == "linux", "GNU backtrace unwinding test requires Linux") +@unittest.skipIf( + _is_arm32_build(), + "GNU backtrace unwinding skipped on Arm 32-bit", +) +class GnuBacktraceUnwindTests(unittest.TestCase): + + def setUp(self): + super().setUp() + try: + _testinternalcapi.gnu_backtrace_unwind() + except RuntimeError as exc: + if "not supported" in str(exc): + self.skipTest("gnu backtrace unwinding not supported on this platform") + raise + + def test_gnu_backtrace_unwinds_through_jit_frames(self): + jit_available = hasattr(sys, "_jit") and sys._jit.is_available() + envs = [({"PYTHON_JIT": "0"}, False)] + if jit_available: + envs.append(({"PYTHON_JIT": "1"}, True)) + + for env, using_jit in envs: + with self.subTest(env=env): + result = _unwind_result("gnu_backtrace_unwind", **env) + python_frames = result.get("python_frames", 0) + jit_frames = result.get("jit_frames", 0) + jit_backend = result.get("jit_backend") + + self.assertGreaterEqual( + python_frames, + STACK_DEPTH, + f"expected to find Python frames in GNU backtrace with env {env}", + ) + if using_jit and jit_backend == "jit": + self.assertGreater( + jit_frames, + 0, + f"expected GNU backtrace to include JIT frames with env {env}", + ) + else: + self.assertEqual( + jit_frames, + 0, + f"unexpected JIT frames counted in GNU backtrace with env {env}", + ) + + def test_gnu_backtrace_jit_frames_disappear_after_executor_free(self): + if not (hasattr(sys, "_jit") and sys._jit.is_available()): + self.skipTest("JIT is not available") + + result = _unwind_after_executor_free_result( + "gnu_backtrace_unwind", PYTHON_JIT="1") + live = result["live"] + if live.get("jit_backend") != "jit": + self.skipTest("JIT backend is not active") + + self.assertGreaterEqual( + live.get("python_frames", 0), + STACK_DEPTH, + "expected live GNU backtrace to include recursive Python frames", + ) + self.assertGreater( + live.get("jit_frames", 0), + 0, + "expected live GNU backtrace to include JIT frames", + ) + + after_free = result["after_free"] + self.assertGreater( + after_free.get("python_frames", 0), + 0, + "expected GNU backtrace after executor free to include Python frames", + ) + self.assertEqual( + after_free.get("jit_frames", 0), + 0, + "unexpected JIT frames in GNU backtrace after executor free", + ) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_abstract.py b/Lib/test/test_capi/test_abstract.py index 3a2ed9f5db82f0f..e455a8636209247 100644 --- a/Lib/test/test_capi/test_abstract.py +++ b/Lib/test/test_capi/test_abstract.py @@ -411,6 +411,11 @@ def test_mapping_getoptionalitem(self): self.assertEqual(getitem(dct2, 'a'), 1) self.assertEqual(getitem(dct2, 'b'), KeyError) + frozendct = frozendict(dct) + self.assertEqual(getitem(frozendct, 'a'), 1) + self.assertEqual(getitem(frozendct, 'b'), KeyError) + self.assertEqual(getitem(frozendct, '\U0001f40d'), 2) + self.assertEqual(getitem(['a', 'b', 'c'], 1), 'b') self.assertRaises(TypeError, getitem, 42, 'a') @@ -431,6 +436,11 @@ def test_mapping_getoptionalitemstring(self): self.assertEqual(getitemstring(dct2, b'a'), 1) self.assertEqual(getitemstring(dct2, b'b'), KeyError) + frozendct = frozendict(dct) + self.assertEqual(getitemstring(frozendct, 'a'), 1) + self.assertEqual(getitemstring(frozendct, 'b'), KeyError) + self.assertEqual(getitemstring(frozendct, '\U0001f40d'.encode()), 2) + self.assertRaises(TypeError, getitemstring, 42, b'a') self.assertRaises(UnicodeDecodeError, getitemstring, {}, b'\xff') self.assertRaises(SystemError, getitemstring, {}, NULL) @@ -677,8 +687,10 @@ def items(self): dict_obj = {'foo': 1, 'bar': 2, 'spam': 3} for mapping in [{}, OrderedDict(), Mapping1(), Mapping2(), + frozendict(), dict_obj, OrderedDict(dict_obj), - Mapping1(dict_obj), Mapping2(dict_obj)]: + Mapping1(dict_obj), Mapping2(dict_obj), + frozendict(dict_obj)]: self.assertListEqual(_testlimitedcapi.mapping_keys(mapping), list(mapping.keys())) self.assertListEqual(_testlimitedcapi.mapping_values(mapping), diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index f10ad50d3bea7e9..c750a6c2a477abd 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -356,9 +356,11 @@ def expect_bool_not(value): for value in new_values: expected, expect_flag = expect_func(value) + old_flags = sys.flags config_set(name, value) self.assertEqual(config_get(name), expected) self.assertEqual(getattr(sys.flags, sys_flag), expect_flag) + self.assertIsNot(sys.flags, old_flags) if name == "write_bytecode": self.assertEqual(getattr(sys, "dont_write_bytecode"), expect_flag) diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 0b2473bac2be113..bbc09e50eb8e450 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -13,7 +13,7 @@ try: import _testinternalcapi except ImportError: - _testinternalcapi = NULL + _testinternalcapi = None # > How about the following counterproposal. This also changes some of # > the other format codes to be a little more regular. diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 4c16bbd4cb0acf1..6d84f0b8c305dfb 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -923,8 +923,8 @@ def test_tp_bases_slot(self): def test_tp_bases_slot_none(self): self.assertRaisesRegex( - SystemError, - "Py_tp_bases is not a tuple", + TypeError, + "bases must be types", _testcapi.create_heapctype_with_none_bases_slot ) @@ -1055,13 +1055,13 @@ def test_heaptype_relative_members(self): def test_heaptype_relative_members_errors(self): with self.assertRaisesRegex( SystemError, - r"With Py_RELATIVE_OFFSET, basicsize must be negative"): + r"With Py_RELATIVE_OFFSET, basicsize must be extended"): _testlimitedcapi.make_heaptype_with_member(0, 1234, 0, True) with self.assertRaisesRegex( - SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + SystemError, r"Member offset out of range \(0\.\.extra_basicsize\)"): _testlimitedcapi.make_heaptype_with_member(0, -8, 1234, True) with self.assertRaisesRegex( - SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + SystemError, r"Member offset must not be negative"): _testlimitedcapi.make_heaptype_with_member(0, -8, -1, True) Sub = _testlimitedcapi.make_heaptype_with_member(0, -8, 0, True) @@ -1078,7 +1078,7 @@ def test_heaptype_relative_special_members_errors(self): with self.subTest(member_name=member_name): with self.assertRaisesRegex( SystemError, - r"With Py_RELATIVE_OFFSET, basicsize must be negative."): + r"With Py_RELATIVE_OFFSET, basicsize must be extended"): _testlimitedcapi.make_heaptype_with_member( basicsize=sys.getsizeof(object()) + 100, add_relative_flag=True, @@ -1089,7 +1089,7 @@ def test_heaptype_relative_special_members_errors(self): ) with self.assertRaisesRegex( SystemError, - r"Member offset out of range \(0\.\.-basicsize\)"): + r"Member offset must not be negative"): _testlimitedcapi.make_heaptype_with_member( basicsize=-8, add_relative_flag=True, @@ -1098,6 +1098,17 @@ def test_heaptype_relative_special_members_errors(self): member_type=_testlimitedcapi.Py_T_PYSSIZET, member_flags=_testlimitedcapi.Py_READONLY, ) + with self.assertRaisesRegex( + SystemError, + r"Member offset out of range \(0\.\.extra_basicsize\)"): + _testlimitedcapi.make_heaptype_with_member( + basicsize=-8, + add_relative_flag=True, + member_name=member_name, + member_offset=1234, + member_type=_testlimitedcapi.Py_T_PYSSIZET, + member_flags=_testlimitedcapi.Py_READONLY, + ) with self.assertRaisesRegex( SystemError, r"type of %s must be Py_T_PYSSIZET" % member_name): diff --git a/Lib/test/test_capi/test_module.py b/Lib/test/test_capi/test_module.py index c32ca1098edc56b..29e1ce5b9af87f5 100644 --- a/Lib/test/test_capi/test_module.py +++ b/Lib/test/test_capi/test_module.py @@ -150,11 +150,13 @@ def test_repeated_def_slot(self): with self.assertRaises(SystemError) as cm: _testcapi.module_from_slots_repeat_slot(spec) self.assertIn(name, str(cm.exception)) - self.assertIn("more than one", str(cm.exception)) + self.assertRegex( + str(cm.exception), + rf"^module( [_\w]+)? has multiple {name}( \(\d+\))? slots$") def test_null_def_slot(self): """Slots that replace PyModuleDef fields can't be NULL""" - for name in (*DEF_SLOTS, 'Py_mod_exec'): + for name in {*DEF_SLOTS, 'Py_mod_exec'} - {'Py_mod_state_size'}: with self.subTest(name): spec = FakeSpec() spec._test_slot_id = getattr(_testcapi, name) diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 635deaa73f7efab..433afac875aa7bf 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -71,6 +71,8 @@ def test_pysentinel_new(self): self.assertIs(type(marker), sentinel) self.assertTrue(_testcapi.pysentinel_check(marker)) self.assertFalse(_testcapi.pysentinel_check(object())) + self.assertTrue(_testcapi.pysentinel_checkexact(marker)) + self.assertFalse(_testcapi.pysentinel_checkexact(object())) self.assertEqual(marker.__name__, "CAPI_SENTINEL") self.assertEqual(marker.__module__, __name__) self.assertEqual(repr(marker), "CAPI_SENTINEL") @@ -80,6 +82,12 @@ def test_pysentinel_new(self): self.assertEqual(no_module.__name__, "NO_MODULE") self.assertIs(no_module.__module__, None) + with_repr = _testcapi.pysentinel_new("WITH_REPR", __name__, "custom repr") + self.assertIs(type(with_repr), sentinel) + self.assertEqual(with_repr.__name__, "WITH_REPR") + self.assertEqual(with_repr.__module__, __name__) + self.assertEqual(repr(with_repr), "custom repr") + globals()["CAPI_SENTINEL"] = marker self.addCleanup(globals().pop, "CAPI_SENTINEL", None) self.assertIs(pickle.loads(pickle.dumps(marker)), marker) @@ -170,11 +178,17 @@ class EnableDeferredRefcountingTest(unittest.TestCase): @support.requires_resource("cpu") def test_enable_deferred_refcount(self): from threading import Thread + import gc self.assertEqual(_testcapi.pyobject_enable_deferred_refcount("not tracked"), 0) foo = [] self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(foo), int(support.Py_GIL_DISABLED)) + # The object must be tracked by the GC + not_gc_tracked = tuple([1, 2]) + self.assertFalse(gc.is_tracked(not_gc_tracked)) + self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(not_gc_tracked), 0) + # Make sure reference counting works on foo now self.assertEqual(foo, []) if support.Py_GIL_DISABLED: diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index e371070d0b7c1c1..2248920c266aef5 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -599,6 +599,7 @@ def testfunc(n, m): self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) + self.assertNotIn("_ITER_NEXT_INLINE", uops) @requires_specialization @@ -1461,7 +1462,132 @@ def testfunc(n): res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertEqual(res, TIER2_THRESHOLD * (TIER2_THRESHOLD - 1) // 2) self.assertIsNotNone(ex) - self.assertIn("_FOR_ITER_TIER_TWO", get_opnames(ex)) + self.assertIn("_ITER_NEXT_INLINE", get_opnames(ex)) + + def test_for_iter_direct_dict_items(self): + def testfunc(n): + d = {i: i * 2 for i in range(10)} + total = 0 + for _ in range(n): + for k, v in d.items(): + total += k + v + return total + + expected = 0 + d = {i: i * 2 for i in range(10)} + for _ in range(TIER2_THRESHOLD): + for k, v in d.items(): + expected += k + v + + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, expected) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertNotIn("_FOR_ITER_TIER_TWO", uops) + + def test_for_iter_direct_dict_keys(self): + def testfunc(n): + d = {i: i for i in range(10)} + total = 0 + for _ in range(n): + for k in d.keys(): + total += k + return total + + expected = TIER2_THRESHOLD * sum(range(10)) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, expected) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertNotIn("_FOR_ITER_TIER_TWO", uops) + + def test_for_iter_direct_dict_values(self): + def testfunc(n): + d = {i: i * 3 for i in range(10)} + total = 0 + for _ in range(n): + for v in d.values(): + total += v + return total + + expected = TIER2_THRESHOLD * sum(i * 3 for i in range(10)) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, expected) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertNotIn("_FOR_ITER_TIER_TWO", uops) + + def test_for_iter_direct_set(self): + def testfunc(n): + s = set(range(10)) + total = 0 + for _ in range(n): + for x in s: + total += x + return total + + expected = TIER2_THRESHOLD * sum(range(10)) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, expected) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertNotIn("_FOR_ITER_TIER_TWO", uops) + + def test_for_iter_direct_reversed(self): + def testfunc(n): + lst = list(range(10)) + total = 0 + for _ in range(n): + for x in reversed(lst): + total += x + return total + + expected = TIER2_THRESHOLD * sum(range(10)) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, expected) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertNotIn("_FOR_ITER_TIER_TWO", uops) + + def test_for_iter_direct_enumerate(self): + def testfunc(n): + lst = list(range(10)) + total = 0 + for _ in range(n): + for i, x in enumerate(lst): + total += i + x + return total + + expected = TIER2_THRESHOLD * sum(i + x for i, x in enumerate(range(10))) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, expected) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertNotIn("_FOR_ITER_TIER_TWO", uops) + + def test_for_iter_direct_zip(self): + def testfunc(n): + a = list(range(10)) + b = list(range(10, 20)) + total = 0 + for _ in range(n): + for x, y in zip(a, b): + total += x + y + return total + + expected = TIER2_THRESHOLD * sum(x + y for x, y in zip(range(10), range(10, 20))) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) + self.assertEqual(res, expected) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertNotIn("_FOR_ITER_TIER_TWO", uops) def test_modified_local_is_seen_by_optimized_code(self): l = sys._getframe().f_locals @@ -3028,7 +3154,7 @@ def testfunc(n): uops = get_opnames(ex) self.assertNotIn("_CHECK_IS_NOT_PY_CALLABLE_KW", uops) - def test_call_len_string(self): + def test_call_len_string_frozen_set_dict(self): def testfunc(n): for _ in range(n): _ = len("abc") @@ -3036,12 +3162,14 @@ def testfunc(n): _ = len(d) _ = len(b"def") _ = len(b"") + _ = len(FROZEN_SET_CONST) + _ = len(FROZEN_DICT_CONST) _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD) self.assertIsNotNone(ex) uops = get_opnames(ex) self.assertNotIn("_CALL_LEN", uops) - self.assertGreaterEqual(count_ops(ex, "_LOAD_CONST_INLINE_BORROW"), 8) + self.assertGreaterEqual(count_ops(ex, "_LOAD_CONST_INLINE_BORROW"), 10) def test_call_len_known_length_small_int(self): # Make sure that len(t) is optimized for a tuple of length 5. @@ -5399,6 +5527,24 @@ def testfunc(n): # _POP_TOP_NOP is a sign the optimizer ran and didn't hit bottom. self.assertGreaterEqual(count_ops(ex, "_POP_TOP_NOP"), 1) + def test_send_virtual(self): + + def send_list(n): + yield from list(range(n)) + def testfunc(n): + for _ in send_list(n): + pass + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + # Ensure SEND is specialized to SEND_VIRTUAL + send_list(10) + res, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD*2) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + + self.assertIn("_FOR_ITER_GEN_FRAME", uops) + self.assertIn("_SEND_VIRTUAL_TIER_TWO", uops) + def test_binary_op_subscr_init_frame(self): class B: def __getitem__(self, other): @@ -5778,6 +5924,44 @@ def __next__(self): """), PYTHON_JIT="1", PYTHON_JIT_STRESS="1") self.assertEqual(result[0].rc, 0, result) + def test_149335_trace_buffer_guard(self): + # https://github.com/python/cpython/issues/149335 + + result = script_helper.run_python_until_end('-c', textwrap.dedent(""" + import sys + + def f1(): + for i_3178 in 0, 2, 10: + mv162 = 162 + + mv3 = mv1 = mv_165 = mv16 = \ + mv167 = mv168 = \ + mv169 = \ + mv_1403_170 = \ + 169 + + mv_1403_170 + + mv_172 = mv_3 = mv_4 = mv175 = mv176 = mv17 = mv178 = mv179 = mv0 = mv1 = mv182 = ( + mv3 + ) = mv4 = mv185 = mv186 = mv187 = mv18 = mv189 = mv0 = mv1 = mv192 = mv3 = mv4 = ( + mv195 + ) = mv196 = mv197 = mv_198 = mv19 = mv0 = mv1 = mv2 = mv3 = mv4 = mv05 = mv06 = ( + mv07 + ) = mv08 = mv09 = mv0 = mv1 = mv2 = mv3 = mv4 = mv15 = mv16 = mv17 = mv18 = mv19 = ( + mv0 + ) = mv1 = mv_2 = mv3 = mv4 = mv_25 = mv_26 = mv_27 = mv_28 = mv_29 = mv0 = mv1 = ( + mv2 + ) = mv_1403 = mv4 = mv35 = mv36 = mv37 = mv38 = mv39 = mv0 = -sys.maxsize / 3 + + mv1 = mv_12 = mv3 = mv_14 = mv45 = sys.float_info.epsilon + mv46 = sys.float_info.epsilon + + for i in range(15000): + f1() + """), PYTHON_JIT="1") + self.assertEqual(result[0].rc, 0, result) + def test_144068_daemon_thread_jit_cleanup(self): result = script_helper.run_python_until_end('-c', textwrap.dedent(""" import threading @@ -5994,6 +6178,20 @@ def __init__(self, x): C(0) if i else str(0) """)) + def test_load_special_type_guard_deopt(self): + script_helper.assert_python_ok("-s", "-c", textwrap.dedent(f""" + def f1(): + class Context: + def __enter__(self): ... + def __exit__(self, e, v, t): ... + + with Context(): + pass + + for _ in range({TIER2_THRESHOLD + 5}): + f1() + """), PYTHON_JIT="1") + def global_identity(x): return x diff --git a/Lib/test/test_capi/test_slots.py b/Lib/test/test_capi/test_slots.py new file mode 100644 index 000000000000000..b8b6d00b5f84d55 --- /dev/null +++ b/Lib/test/test_capi/test_slots.py @@ -0,0 +1,349 @@ +from test.support import import_helper, subTests +import contextlib +import unittest +import types +import sys + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +_testcapi = import_helper.import_module('_testcapi') + +class FakeSpec: + name = 'module' + + +# See Modules/_testlimitedcapi/slots.c for the definitions. +# This module is full of "magic constants" which simply need to match +# between the C and Python part of the tests. + +class TypeSlotsTests(unittest.TestCase): + def test_basic_type_slots(self): + cls = _testlimitedcapi.type_from_slots("basic") + self.assertEqual(cls.__name__, "MyType") + + # Py_TPFLAGS_IMMUTABLETYPE is *not* set + cls.attr = 123 + + # Py_TPFLAGS_BASETYPE is *not* set + with self.assertRaises(TypeError): + class Sub(cls): + pass + + def test_mod_slot_in_type(self): + with self.assertRaisesRegex(SystemError, "invalid.* 100 .*Py_mod_name"): + _testlimitedcapi.type_from_slots("foreign_slot") + + def test_size_slots(self): + cls = _testlimitedcapi.type_from_slots("basicsize") + self.assertGreaterEqual(cls.__basicsize__, 256) + + cls = _testlimitedcapi.type_from_slots("extra_basicsize") + self.assertGreaterEqual(cls.__basicsize__, object.__basicsize__ + 256) + + cls = _testlimitedcapi.type_from_slots("itemsize") + self.assertGreaterEqual(cls.__itemsize__, 16) + + def test_flag_slots(self): + cls = _testlimitedcapi.type_from_slots("flags") + with self.assertRaises(TypeError): + # Py_TPFLAGS_IMMUTABLETYPE is set + cls.attr = 123 + class Sub(cls): + # Py_TPFLAGS_BASETYPE is set + pass + + def test_func_slots(self): + cls = _testlimitedcapi.type_from_slots("matmul_123") + self.assertEqual(cls() @ None, 123) + + def test_optional_end(self): + with self.assertRaisesRegex(SystemError, "invalid flags.*Py_slot_end"): + cls = _testlimitedcapi.type_from_slots("optional_end") + + def test_invalid(self): + with self.assertRaisesRegex(SystemError, "Py_slot_invalid"): + cls = _testlimitedcapi.type_from_slots("invalid") + with self.assertRaisesRegex(SystemError, f"slot ID {0xfbad}"): + cls = _testlimitedcapi.type_from_slots("invalid_fbad") + + cls = _testlimitedcapi.type_from_slots("optional_invalid") + self.assertGreaterEqual(cls.__basicsize__, object.__basicsize__ + 256) + + cls = _testlimitedcapi.type_from_slots("optional_invalid_fbad") + self.assertGreaterEqual(cls.__basicsize__, object.__basicsize__ + 256) + + @subTests("case_name", ["old_slot_numbers", "new_slot_numbers"]) + def test_compat_slot_numbers(self, case_name): + cls = _testlimitedcapi.type_from_slots(case_name) + obj = cls() + + # Py_bf_getbuffer (1), Py_bf_releasebuffer (2) + self.assertEqual(obj.buf_counter, 0) + mem = memoryview(obj) + self.assertEqual(bytes(mem), b"buf\0") + self.assertEqual(obj.buf_counter, 1) + mem.release() + self.assertEqual(obj.buf_counter, 0) + + # Py_mp_ass_subscript (3) + with self.assertRaises(KeyError): + obj["key"] = "value" + + # Py_mp_length (4) + self.assertEqual(len(obj), 456) + + def test_nonstatic_tp_members(self): + with self.assertRaisesRegex(SystemError, "Py_tp_members.*STATIC"): + _testlimitedcapi.type_from_slots("nonstatic_tp_members") + + def test_intptr_flags(self): + cls = _testlimitedcapi.type_from_slots("intptr_flags_macro") + with self.assertRaises(TypeError): + # Py_TPFLAGS_IMMUTABLETYPE is set + cls.attr = 123 + + cls = _testlimitedcapi.type_from_slots("intptr_flags_struct") + with self.assertRaises(TypeError): + # Py_TPFLAGS_IMMUTABLETYPE is set + cls.attr = 123 + + cls = _testlimitedcapi.type_from_slots("intptr_static") + cls.attribute = 123 + + def test_nested(self): + cls = _testlimitedcapi.type_from_slots("nested") + self.assertEqual(cls() + 1, 123) + self.assertEqual(cls() - 1, 234) + + cls = _testlimitedcapi.type_from_slots("nested_max") + self.assertEqual(cls() + 1, 123) + self.assertEqual(cls() - 1, 234) + self.assertEqual(cls() * 1, 345) + self.assertEqual(cls() / 1, 456) + self.assertEqual(cls() % 1, 567) + + with self.assertRaisesRegex(SystemError, "too many levels"): + _testlimitedcapi.type_from_slots("nested_over_limit") + + cls = _testlimitedcapi.type_from_slots("nested_old") + self.assertEqual(cls() + 1, 123) + self.assertEqual(cls() - 1, 234) + + cls = _testlimitedcapi.type_from_slots("nested_old_max") + self.assertEqual(cls() + 1, 123) + self.assertEqual(cls() - 1, 234) + self.assertEqual(cls() * 1, 345) + self.assertEqual(cls() / 1, 456) + self.assertEqual(cls() % 1, 567) + + with self.assertRaisesRegex(SystemError, "too many levels"): + _testlimitedcapi.type_from_slots("nested_old_over_limit") + + cls = _testlimitedcapi.type_from_slots("nested_pingpong") + self.assertEqual(cls() + 1, 123) + self.assertEqual(cls() - 1, 234) + self.assertEqual(cls() * 1, 345) + self.assertEqual(cls() / 1, 456) + self.assertEqual(cls() % 1, 567) + + # Slot names aren't exposed to Python yet; see Include/slots_generated.h + # for the definitions. + + @subTests("slot_number", [ + *range(1, 83), # Original slots + *range(88, 92), # New compat slot values + *range(95, 99), # Slots for PyType_Spec fields + *range(107, 109), # Slots for PyType_FromMetaclass args + ]) + def test_null_slot_handling(self, slot_number): + if slot_number == 56: + # Py_tp_doc + return + elif slot_number == 72 or slot_number >= 95: + # Py_tp_members; all new slots + ctx = self.assertRaisesRegex( + SystemError, "NULL not allowed|must be positive") + ctx_old = ctx + else: + ctx = self.assertWarnsRegex(DeprecationWarning, "NULL") + ctx_old = contextlib.nullcontext() + with ctx: + _testlimitedcapi.type_from_null_slot(slot_number) + if slot_number < 95: + with ctx_old: + _testlimitedcapi.type_from_null_spec_slot(slot_number) + + def test_repeat_warning(self): + with self.assertWarnsRegex(DeprecationWarning, "multiple"): + cls = _testlimitedcapi.type_from_slots("repeat_add") + self.assertEqual(cls() + 1, 456) + + def test_repeat_error(self): + with self.assertRaisesRegex(SystemError, "multiple"): + cls = _testlimitedcapi.type_from_slots("repeat_module") + +class ModuleSlotsTests(unittest.TestCase): + def test_basic_module_slots(self): + mod = _testlimitedcapi.module_from_slots("basic", FakeSpec()) + self.assertIsInstance(mod, types.ModuleType) + + def test_type_slot_in_module(self): + with self.assertRaisesRegex(SystemError, "invalid.* 95 .*Py_tp_name"): + _testlimitedcapi.module_from_slots("foreign_slot", FakeSpec()) + + def test_size_slots(self): + mod = _testlimitedcapi.module_from_slots("state_size", FakeSpec()) + self.assertEqual(mod.state_size, 42) + + def test_flag_slots(self): + mod = _testlimitedcapi.module_from_slots("multi_interp", FakeSpec()) + + def test_exec_slot(self): + mod = _testlimitedcapi.module_from_slots("exec", FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + + def test_optional_end(self): + with self.assertRaisesRegex(SystemError, "invalid flags.*Py_slot_end"): + _testlimitedcapi.module_from_slots("optional_end", FakeSpec()) + + def test_invalid(self): + with self.assertRaisesRegex(SystemError, "Py_slot_invalid"): + _testlimitedcapi.module_from_slots("invalid", FakeSpec()) + with self.assertRaisesRegex(SystemError, f"slot ID {0xfbad}"): + _testlimitedcapi.module_from_slots("invalid_fbad", FakeSpec()) + + mod = _testlimitedcapi.module_from_slots("optional_invalid", FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + + mod = _testlimitedcapi.module_from_slots("optional_invalid_fbad", FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + + @subTests("case_name", ["old_slot_numbers", "new_slot_numbers"]) + def test_compat_slot_numbers(self, case_name): + mod = _testlimitedcapi.module_from_slots(case_name, FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + + @subTests("case_name", ["old_slot_number_create", "new_slot_number_create"]) + def test_compat_slot_number_create(self, case_name): + spec = FakeSpec() + mod = _testlimitedcapi.module_from_slots(case_name, spec) + self.assertIs(mod, spec) + + @subTests("slot_number", [4, 87]) + def test_compat_slot_number_gil(self, slot_number): + spec = FakeSpec() + gil_enabled = sys._is_gil_enabled() + mod = _testlimitedcapi.module_from_gil_slot(slot_number, spec) + self.assertEqual(gil_enabled, sys._is_gil_enabled()) + + def test_nonstatic_mod_methods(self): + with self.assertRaisesRegex(SystemError, "Py_mod_methods.*STATIC"): + _testlimitedcapi.module_from_slots("nonstatic_mod_methods", + FakeSpec()) + + def test_intptr_methods(self): + mod = _testlimitedcapi.module_from_slots("intptr_methods", + FakeSpec()) + self.assertEqual(mod.type_from_slots.__name__, "type_from_slots") + + def test_nested(self): + mod = _testlimitedcapi.module_from_slots("nested", FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + self.assertEqual(mod.__doc__, "doc") + + mod = _testlimitedcapi.module_from_slots("nested_max", FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + self.assertEqual(mod.state_size, 53) + self.assertEqual(mod.__doc__, "doc") + + with self.assertRaisesRegex(SystemError, "too many levels"): + _testlimitedcapi.module_from_slots("nested_over_limit", FakeSpec()) + + mod = _testlimitedcapi.module_from_slots("nested_old", FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + self.assertEqual(mod.__doc__, "doc") + + mod = _testlimitedcapi.module_from_slots("nested_old_max", FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + self.assertEqual(mod.state_size, 53) + self.assertEqual(mod.__doc__, "doc") + + with self.assertRaisesRegex(SystemError, "too many levels"): + _testlimitedcapi.module_from_slots("nested_old_over_limit", FakeSpec()) + + mod = _testlimitedcapi.module_from_slots("nested_pingpong", FakeSpec()) + self.assertEqual(mod.exec_done, "yes") + self.assertEqual(mod.state_size, 53) + self.assertEqual(mod.__doc__, "doc") + + def test_nested_nonstatic_from_def(self): + with self.assertRaisesRegex(SystemError, "must be static"): + _testcapi.module_from_def_nonstatic_nested(FakeSpec()) + + # Slot names aren't exposed to Python yet; see Include/slots_generated.h + # for the definitions. + + @subTests("slot_number", [ + *range(1, 5), # Old compat slot values + *range(84, 88), # New compat slot values + *range(100, 107), # Slots for PyModuleDef fields + *range(109, 111), # Slots new in 3.15 + ]) + def test_null_slot_handling(self, slot_number): + if slot_number in {3, 86, 4, 87, 102}: + # Py_mod_mult.interp., Py_mod_gil, Py_mod_state_size + return + elif slot_number in {2, 85} or slot_number > 85: + # Py_mod_exec, new slots + ctx = self.assertRaisesRegex(SystemError, "NULL not allowed") + ctx_old = ctx + else: + ctx = self.assertWarnsRegex(DeprecationWarning, "NULL") + ctx_old = contextlib.nullcontext() + with ctx: + _testlimitedcapi.module_from_null_slot(slot_number, FakeSpec()) + with ctx_old: + _testcapi.module_from_null_def_slot(slot_number, + FakeSpec()) + + def test_repeat_error(self): + with self.assertRaisesRegex(SystemError, "multiple"): + _testlimitedcapi.module_from_slots("repeat_create", FakeSpec()) + with self.assertRaisesRegex(SystemError, "multiple"): + _testlimitedcapi.module_from_slots("repeat_exec", FakeSpec()) + with self.assertRaisesRegex(SystemError, "multiple"): + _testlimitedcapi.module_from_slots("repeat_gil", FakeSpec()) + + def test_bases_slots(self): + create = _testlimitedcapi.type_from_base_slots + + # Py_tp_bases overrides Py_tp_base + cls = create(base=int, bases=float) + self.assertEqual(cls.mro(), [cls, float, object]) + + # type is equivalent to one-element tuple + cls = create(base=None, bases=int) + self.assertEqual(cls.mro(), [cls, int, object]) + + cls = create(base=None, bases=(int,)) + self.assertEqual(cls.mro(), [cls, int, object]) + + cls = create(base=int) + self.assertEqual(cls.mro(), [cls, int, object]) + + cls = create(base=(int,)) + self.assertEqual(cls.mro(), [cls, int, object]) + + # Tuple of bases works + class Custom: + pass + cls = create(bases=int) + sub = create(base=float, bases=(Custom, cls, int)) + self.assertEqual(sub.mro(), [sub, Custom, cls, int, object]) + + # Reasonable error message for non-types + with self.assertRaisesRegex(TypeError, + "bases must be types; got 'NoneType'"): + create(base=None) + with self.assertRaisesRegex(TypeError, + "bases must be types; got 'str'"): + create(bases="a string") diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 5dee25756fe2893..0dcd8a25ad0128d 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -842,9 +842,7 @@ def test_fromwidechar(self): if SIZEOF_WCHAR_T == 2: self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d') - self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX) self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2) - self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN) self.assertEqual(fromwidechar(NULL, 0), '') self.assertRaises(SystemError, fromwidechar, NULL, 1) self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MAX) @@ -852,6 +850,10 @@ def test_fromwidechar(self): self.assertRaises(SystemError, fromwidechar, NULL, -2) self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) + # The following tests are skipped since they rely on undefined behavior + #self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX) + #self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN) + @support.cpython_only @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidechar(self): diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 67595e3550b0ff2..490ae7b23e62795 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -208,6 +208,7 @@ class TestTypeWatchers(unittest.TestCase): TYPES = 0 # appends modified types to global event list ERROR = 1 # unconditionally sets and signals a RuntimeException WRAP = 2 # appends modified type wrapped in list to global event list + NAME = 3 # appends type name (string) to global event list # duplicating the C constant TYPE_MAX_WATCHERS = 8 @@ -377,6 +378,27 @@ def test_clear_unassigned_watcher_id(self): with self.assertRaisesRegex(ValueError, r"No type watcher set for ID 1"): self.clear_watcher(1) + def test_watch_type_dealloc(self): + # Use the NAME watcher (kind=3) which records the type's name as a + # string, avoiding any reference to the type object itself during + # deallocation. + with self.watcher(kind=self.NAME) as wid: + class MyTestType: pass + self.watch(wid, MyTestType) + del MyTestType + gc_collect() + events = _testcapi.get_type_modified_events() + self.assertIn("MyTestType", events) + + def test_watch_type_dealloc_error(self): + with self.watcher(kind=self.ERROR) as wid: + class MyTestType2: pass + self.watch(wid, MyTestType2) + with catch_unraisable_exception() as cm: + del MyTestType2 + gc_collect() + self.assertEqual(str(cm.unraisable.exc_value), "boom!") + def test_no_more_ids_available(self): with self.assertRaisesRegex(RuntimeError, r"no more type watcher IDs"): with ExitStack() as stack: diff --git a/Lib/test/test_capi/test_weakref.py b/Lib/test/test_capi/test_weakref.py new file mode 100644 index 000000000000000..7c9e97f571979dc --- /dev/null +++ b/Lib/test/test_capi/test_weakref.py @@ -0,0 +1,138 @@ +import weakref +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +NULL = None + +class Object: + pass + +class Ref(weakref.ReferenceType): + pass + + +class CAPIWeakrefTest(unittest.TestCase): + def test_pyweakref_check(self): + # Test PyWeakref_Check() + check = _testlimitedcapi.pyweakref_check + obj = Object() + self.assertEqual(check(obj), 0) + self.assertEqual(check(weakref.ref(obj)), 1) + self.assertEqual(check(Ref(obj)), 1) + self.assertEqual(check(weakref.proxy(obj)), 1) + + # CRASHES check(NULL) + + def test_pyweakref_checkref(self): + # Test PyWeakref_CheckRef() + checkref = _testlimitedcapi.pyweakref_checkref + obj = Object() + self.assertEqual(checkref(obj), 0) + self.assertEqual(checkref(weakref.ref(obj)), 1) + self.assertEqual(checkref(Ref(obj)), 1) + self.assertEqual(checkref(weakref.proxy(obj)), 0) + + # CRASHES checkref(NULL) + + def test_pyweakref_checkrefexact(self): + # Test PyWeakref_CheckRefExact() + checkrefexact = _testlimitedcapi.pyweakref_checkrefexact + obj = Object() + self.assertEqual(checkrefexact(obj), 0) + self.assertEqual(checkrefexact(weakref.ref(obj)), 1) + self.assertEqual(checkrefexact(Ref(obj)), 0) + self.assertEqual(checkrefexact(weakref.proxy(obj)), 0) + + # CRASHES checkrefexact(NULL) + + def test_pyweakref_checkproxy(self): + # Test PyWeakref_CheckProxy() + checkproxy = _testlimitedcapi.pyweakref_checkproxy + obj = Object() + self.assertEqual(checkproxy(obj), 0) + self.assertEqual(checkproxy(weakref.ref(obj)), 0) + self.assertEqual(checkproxy(Ref(obj)), 0) + self.assertEqual(checkproxy(weakref.proxy(obj)), 1) + + # CRASHES checkproxy(NULL) + + def test_pyweakref_getref(self): + # Test PyWeakref_GetRef() + getref = _testcapi.pyweakref_getref + obj = Object() + wr = weakref.ref(obj) + wp = weakref.proxy(obj) + self.assertEqual(getref(wr), (1, obj)) + self.assertEqual(getref(wp), (1, obj)) + del obj + self.assertEqual(getref(wr), 0) + self.assertEqual(getref(wp), 0) + + self.assertRaises(TypeError, getref, 42) + self.assertRaises(SystemError, getref, NULL) + + def test_pyweakref_isdead(self): + # Test PyWeakref_IsDead() + isdead = _testcapi.pyweakref_isdead + obj = Object() + wr = weakref.ref(obj) + wp = weakref.proxy(obj) + self.assertEqual(isdead(wr), 0) + self.assertEqual(isdead(wp), 0) + del obj + self.assertEqual(isdead(wr), 1) + self.assertEqual(isdead(wp), 1) + + self.assertRaises(TypeError, isdead, 42) + self.assertRaises(SystemError, isdead, NULL) + + def test_pyweakref_newref(self): + # Test PyWeakref_NewRef() + newref = _testlimitedcapi.pyweakref_newref + obj = Object() + wr = newref(obj) + self.assertIs(type(wr), weakref.ReferenceType) + # PyWeakref_NewRef() handles None callback as NULL callback + wr = newref(obj, None) + self.assertIs(type(wr), weakref.ReferenceType) + self.assertRaises(TypeError, newref, obj, 42) + log = [] + wr = newref(obj, log.append) + self.assertIs(type(wr), weakref.ReferenceType) + self.assertEqual(log, []) + del obj + self.assertEqual(log, [wr]) + + self.assertRaises(TypeError, newref, []) + # CRASHES newref(NULL) + + def test_pyweakref_newproxy(self): + # Test PyWeakref_NewProxy() + newproxy = _testlimitedcapi.pyweakref_newproxy + obj = Object() + wp = newproxy(obj) + self.assertIs(type(wp), weakref.ProxyType) + # PyWeakref_NewProxy() handles None callback as NULL callback + wp = newproxy(obj, None) + self.assertIs(type(wp), weakref.ProxyType) + self.assertRaises(TypeError, newproxy, obj, 42) + log = [] + wp = newproxy(obj, log.append) + self.assertIs(type(wp), weakref.ProxyType) + self.assertEqual(log, []) + del obj + self.assertEqual(log, [wp]) + + def func(): + pass + wp = newproxy(func) + self.assertIs(type(wp), weakref.CallableProxyType) + + self.assertRaises(TypeError, newproxy, []) + # CRASHES newproxy(NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 1958c44e2b64ef7..4cc5f843dd388da 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -8,6 +8,8 @@ import shlex import shutil import subprocess +import sysconfig +import sys import unittest from test import support @@ -62,6 +64,9 @@ def run_cmd(operation, cmd): env['CPYTHON_TEST_LIMITED'] = '1' if abi3t: env['CPYTHON_TEST_ABI3T'] = '1' + if support.MS_WINDOWS and sysconfig.is_python_build(): + env['CPYTHON_EXTRA_INCDIRS'] = os.path.split(sysconfig.get_config_h_filename())[0] + env['CPYTHON_EXTRA_LIBDIRS'] = os.path.split(sys.executable)[0] env['CPYTHON_TEST_EXT_NAME'] = extension_name env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) if support.verbose: diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index a880cb82811f783..895eca50f03b985 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -99,6 +99,10 @@ _testcext_exec(PyObject *module) obj = NULL; Py_CLEAR(obj); + // Test that Py_BEGIN_CRITICAL_SECTION is available + Py_BEGIN_CRITICAL_SECTION(module); + Py_END_CRITICAL_SECTION(); + return 0; } @@ -121,13 +125,13 @@ _Py_COMP_DIAG_PUSH PyDoc_STRVAR(_testcext_doc, "C test extension."); PyABIInfo_VAR(abi_info); -static PyModuleDef_Slot _testcext_slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, STR(MODULE_NAME)}, - {Py_mod_doc, (void*)(char*)_testcext_doc}, - {Py_mod_exec, (void*)_testcext_exec}, - {Py_mod_methods, _testcext_methods}, - {0, NULL} +static PySlot _testcext_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, STR(MODULE_NAME)), + PySlot_STATIC_DATA(Py_mod_doc, (void*)(char*)_testcext_doc), + PySlot_FUNC(Py_mod_exec, (void*)_testcext_exec), + PySlot_STATIC_DATA(Py_mod_methods, _testcext_methods), + PySlot_END, }; _Py_COMP_DIAG_POP diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 25fe50df603883f..1eca44bdf823dc2 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -1,7 +1,6 @@ # gh-91321: Build a basic C test extension to check that the Python C API is # compatible with C and does not emit C compiler warnings. import os -import platform import shlex import sys import sysconfig @@ -66,6 +65,8 @@ def main(): limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) abi3t = bool(os.environ.get("CPYTHON_TEST_ABI3T", "")) internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) + incdirs = os.environ.get("CPYTHON_EXTRA_INCDIRS", "") + libdirs = os.environ.get("CPYTHON_EXTRA_LIBDIRS", "") sources = [SOURCE] @@ -106,19 +107,16 @@ def main(): if internal: cflags.append('-DTEST_INTERNAL_C_API=1') - # On Windows, add PCbuild\amd64\ to include and library directories + # Add additional include and library directories, typically for in-tree + # testing where not all directories are inferred include_dirs = [] library_dirs = [] - if support.MS_WINDOWS: - srcdir = sysconfig.get_config_var('srcdir') - machine = platform.uname().machine - pcbuild = os.path.join(srcdir, 'PCbuild', machine) - if os.path.exists(pcbuild): - # pyconfig.h is generated in PCbuild\amd64\ - include_dirs.append(pcbuild) - # python313.lib is generated in PCbuild\amd64\ - library_dirs.append(pcbuild) - print(f"Add PCbuild directory: {pcbuild}") + if incdirs: + print("Add incdirs:", incdirs) + include_dirs.extend(incdirs.split(os.pathsep)) + if libdirs: + print("Add libdirs:", libdirs) + library_dirs.extend(libdirs.split(os.pathsep)) # Display information to help debugging for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 8740f65b7b0d1d0..a8645af26b25d87 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -57,6 +57,7 @@ def verify_valid_flag(self, cmd_line): return out @support.cpython_only + @support.force_not_colorized def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -68,6 +69,7 @@ def test_help(self): self.assertLess(len(lines), 50) @support.cpython_only + @support.force_not_colorized def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) @@ -81,6 +83,7 @@ def test_help_env(self): "env vars should be sorted alphabetically") @support.cpython_only + @support.force_not_colorized def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) @@ -89,6 +92,7 @@ def test_help_xoptions(self): "options should be sorted alphabetically") @support.cpython_only + @support.force_not_colorized def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -100,6 +104,25 @@ def test_help_all(self): # but the rest should be ASCII-only b''.join(lines[1:]).decode('ascii') + @support.cpython_only + @support.force_colorized + def test_help_colorized(self): + rc, out, err = assert_python_ok("--help", FORCE_COLOR="1") + # Check ANSI color codes are present + self.assertIn(b"\x1b[", out) + # Check that key text elements are still present + self.assertIn(b"usage:", out) + self.assertIn(b"-h", out) + self.assertIn(b"--help-all", out) + self.assertIn(b"cmd", out) + self.assertIn(b"Arguments:", out) + + @support.cpython_only + @support.force_not_colorized + def test_help_not_colorized(self): + rc, out, err = assert_python_ok("--help") + self.assertNotIn(b"\x1b[", out) + def test_optimize(self): self.verify_valid_flag('-O') self.verify_valid_flag('-OO') @@ -1013,6 +1036,7 @@ def test_python_legacy_windows_stdio(self): p = subprocess.run([sys.executable, "-c", code], creationflags=subprocess.CREATE_NEW_CONSOLE, env=env) + support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) self.assertEqual(p.returncode, 0) # Then test that FIleIO is used when PYTHONLEGACYWINDOWSSTDIO is set. @@ -1021,6 +1045,7 @@ def test_python_legacy_windows_stdio(self): p = subprocess.run([sys.executable, "-c", code], creationflags=subprocess.CREATE_NEW_CONSOLE, env=env) + support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) self.assertEqual(p.returncode, 0) @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), @@ -1063,6 +1088,7 @@ def test_argv0_normalization(self): self.assertEqual(proc.stdout.strip(), b'0') @support.cpython_only + @support.force_not_colorized def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, @@ -1288,6 +1314,17 @@ def test_presite(self): proc = assert_python_ok("-X", f"presite={entrypoint}", "-c", "pass") self.assertEqual(proc.out.rstrip(), b"presite func") + def test_dump_path_config(self): + # gh-151253: At the first import (import encodings) during Python + # startup, if the import fails, dump the Python path configuration. + nonexistent = '/nonexistent-python-path' + # Use -X frozen_modules=off to disable frozen encodings module + # on release build. + cmd = ["-X", "frozen_modules=off", "-c", "pass"] + proc = assert_python_failure(*cmd, PYTHONHOME=nonexistent) + self.assertIn(b'Python path configuration:', proc.err) + self.assertIn(f"PYTHONHOME = '{nonexistent}'".encode(), proc.err) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 3588872ed23ac49..657e70a92f09b9d 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -215,7 +215,6 @@ from test.support import threading_helper, import_helper from test.support.bytecode_helper import instructions_with_positions from opcode import opmap, opname -from _testcapi import code_offset_to_line try: import _testinternalcapi except ModuleNotFoundError: @@ -1491,6 +1490,8 @@ async def async_func(): rc, out, err = assert_python_ok('-OO', '-c', code) def test_co_branches(self): + _testcapi = import_helper.import_module("_testcapi") + code_offset_to_line = _testcapi.code_offset_to_line def get_line_branches(func): code = func.__code__ diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 79c8a7ef8864829..8fdd08df9e4f46a 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1892,9 +1892,11 @@ def test_copy(self): self.assertIsNot(dup, orig) self.assertEqual(dup, orig) self.assertTrue(orig._is_text_encoding) + self.assertIsInstance(orig._expat_decoding_table, tuple) self.assertEqual(dup.encode, orig.encode) self.assertEqual(dup.name, orig.name) self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + self.assertIs(dup._expat_decoding_table, orig._expat_decoding_table) # Test a CodecInfo with _is_text_encoding equal to false. orig = codecs.lookup("base64") @@ -1902,9 +1904,11 @@ def test_copy(self): self.assertIsNot(dup, orig) self.assertEqual(dup, orig) self.assertFalse(orig._is_text_encoding) + self.assertNotHasAttr(orig, '_expat_decoding_table') self.assertEqual(dup.encode, orig.encode) self.assertEqual(dup.name, orig.name) self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + self.assertNotHasAttr(dup, '_expat_decoding_table') def test_deepcopy(self): orig = codecs.lookup('utf-8') @@ -1912,9 +1916,11 @@ def test_deepcopy(self): self.assertIsNot(dup, orig) self.assertEqual(dup, orig) self.assertTrue(orig._is_text_encoding) + self.assertIsInstance(orig._expat_decoding_table, tuple) self.assertEqual(dup.encode, orig.encode) self.assertEqual(dup.name, orig.name) self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + self.assertIs(dup._expat_decoding_table, orig._expat_decoding_table) # Test a CodecInfo with _is_text_encoding equal to false. orig = codecs.lookup("base64") @@ -1922,9 +1928,11 @@ def test_deepcopy(self): self.assertIsNot(dup, orig) self.assertEqual(dup, orig) self.assertFalse(orig._is_text_encoding) + self.assertNotHasAttr(orig, '_expat_decoding_table') self.assertEqual(dup.encode, orig.encode) self.assertEqual(dup.name, orig.name) self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + self.assertNotHasAttr(dup, '_expat_decoding_table') def test_pickle(self): codec_info = codecs.lookup('utf-8') @@ -1940,6 +1948,8 @@ def test_pickle(self): unpickled_codec_info.incrementalencoder ) self.assertTrue(unpickled_codec_info._is_text_encoding) + self.assertEqual(unpickled_codec_info._expat_decoding_table, + codec_info._expat_decoding_table) # Test a CodecInfo with _is_text_encoding equal to false. codec_info = codecs.lookup('base64') @@ -1955,6 +1965,7 @@ def test_pickle(self): unpickled_codec_info.incrementalencoder ) self.assertFalse(unpickled_codec_info._is_text_encoding) + self.assertNotHasAttr(unpickled_codec_info, '_expat_decoding_table') class StreamReaderTest(unittest.TestCase): diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index d02937c84d95349..36058854a41d63e 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -161,3 +161,34 @@ def test_syntax_error__return_not_in_function(self): self.assertIsNone(cm.exception.text) self.assertEqual(cm.exception.offset, 1) self.assertEqual(cm.exception.end_offset, 10) + + def test_frozenset_optimization(self): + l1 = self.Label() + snippet = "frozenset({1, 2, 3})" + expected = [ + ('RESUME', 0), + ('ANNOTATIONS_PLACEHOLDER', None), + ('LOAD_NAME', 0), + ('COPY', 1), + ('LOAD_COMMON_CONSTANT', 12), + ('IS_OP', 0), + ('POP_JUMP_IF_FALSE', l1), + ('POP_TOP', None), + ('LOAD_CONST', 1), + ('LOAD_CONST', 2), + ('LOAD_CONST', 3), + ('BUILD_SET', 3), + ('CALL_INTRINSIC_1', 12), + ('JUMP', 0), + l1, + ('PUSH_NULL', None), + ('LOAD_CONST', 1), + ('LOAD_CONST', 2), + ('LOAD_CONST', 3), + ('BUILD_SET', 3), + ('CALL', 1), + ('POP_TOP', None), + ('LOAD_CONST', 0), + ('RETURN_VALUE', None) + ] + self.codegen_test(snippet, expected) diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index 5ea543bf7489821..ca612db17ce8021 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -147,6 +147,8 @@ def test_spawn(self): self._test(ProcessPoolSpawnFailingInitializerTest) @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + @unittest.skipIf(sys.platform == "cygwin", + "Forkserver is not available on Cygwin") def test_forkserver(self): self._test(ProcessPoolForkserverFailingInitializerTest) diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 731419a48bd1281..45cfd46fb7befbf 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -3,6 +3,7 @@ import sys import threading import time +import traceback import unittest import unittest.mock from concurrent import futures @@ -62,6 +63,32 @@ def test_killed_child(self): # Submitting other jobs fails as well. self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_broken_process_pool_traceback(self): + # When a child process is abruptly terminated, the whole pool gets + # "broken", and a BrokenProcessPool exception should be created + # for each future instead of sharing one exception among all futures. + event = self.create_event() + futures = [self.executor.submit(event.wait) for _ in range(3)] + p = next(iter(self.executor._processes.values())) + p.terminate() + for fut in futures: + # Don't use assertRaises(): it clears the traceback off exc. + try: + fut.result() + except BrokenProcessPool as exc: + tb = exc.__traceback__ + else: + self.fail("BrokenProcessPool not raised") + count = sum( + 1 + for frame_summary in traceback.extract_tb(tb) + if frame_summary.filename == __file__ + ) + # This code file should appear exactly once in the traceback. + # A shared exception would accumulate a frame per result() call. + self.assertEqual(count, 1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() def test_map_chunksize(self): def bad_map(): @@ -115,11 +142,12 @@ def test_traceback_when_child_process_terminates_abruptly(self): with self.assertRaises(BrokenProcessPool) as bpe: future.result() - cause = bpe.exception.__cause__ - self.assertIsInstance(cause, futures.process._RemoteTraceback) - self.assertIn( - f"terminated abruptly with exit code {exit_code}", cause.tb - ) + if sys.platform != 'cygwin': + cause = bpe.exception.__cause__ + self.assertIsInstance(cause, futures.process._RemoteTraceback) + self.assertIn( + f"terminated abruptly with exit code {exit_code}", cause.tb + ) @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @hashlib_helper.requires_hashdigest('md5') diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index 2a9e55152b82d54..006360c8d941c9d 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -135,7 +135,7 @@ def get_context(self): _check_system_limits() except NotImplementedError: self.skipTest("ProcessPoolExecutor unavailable on this system") - if sys.platform == "win32": + if sys.platform in ("win32", "cygwin"): self.skipTest("require unix system") if support.check_sanitizer(thread=True): self.skipTest("TSAN doesn't support threads after fork") diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 8d8dd2a2bf27fbf..4783943f71a1092 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -526,6 +526,17 @@ def test_default_case_sensitivity(self): cf.get(self.default_section, "Foo"), "Bar", "could not locate option, expecting case-insensitive defaults") + def test_crlf_normalization(self): + cf = self.newconfig({"key1": "a\nb","key2": "a\rb", "key3": "a\r\nb", "key4": "a\r\nb"}) + buf = io.StringIO() + cf.write(buf) + cf_str = buf.getvalue() + self.assertNotIn("\r", cf_str) + self.assertNotIn("\r\n", cf_str) + self.assertEqual(cf_str.count("\n"), 10) + self.assertEqual(cf_str.count("\n\t"), 4) + self.assertTrue(cf_str.endswith("\n\n")) + def test_parse_errors(self): cf = self.newconfig() self.parse_error(cf, configparser.ParsingError, diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 5b4c97c181bb6ae..967feee6693c037 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -6,6 +6,7 @@ import shutil import subprocess import sys +import sysconfig import unittest from test import support @@ -50,6 +51,9 @@ def run_cmd(operation, cmd): env['CPYTHON_TEST_CPP_STD'] = std if limited: env['CPYTHON_TEST_LIMITED'] = '1' + if support.MS_WINDOWS and sysconfig.is_python_build(): + env['CPYTHON_EXTRA_INCDIRS'] = os.path.split(sysconfig.get_config_h_filename())[0] + env['CPYTHON_EXTRA_LIBDIRS'] = os.path.split(sys.executable)[0] env['CPYTHON_TEST_EXT_NAME'] = extension_name env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) if extra_cflags: diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index 14aeafefcaa8f72..5d004ca6e3ad788 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -1,7 +1,6 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os -import platform import shlex import sys import sysconfig @@ -48,6 +47,8 @@ def main(): module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) + incdirs = os.environ.get("CPYTHON_EXTRA_INCDIRS", "") + libdirs = os.environ.get("CPYTHON_EXTRA_LIBDIRS", "") cppflags = list(CPPFLAGS) cppflags.append(f'-DMODULE_NAME={module_name}') @@ -90,19 +91,16 @@ def main(): if extra_cflags: cppflags.extend(shlex.split(extra_cflags)) - # On Windows, add PCbuild\amd64\ to include and library directories + # Add additional include and library directories, typically for in-tree + # testing where not all directories are inferred include_dirs = [] library_dirs = [] - if support.MS_WINDOWS: - srcdir = sysconfig.get_config_var('srcdir') - machine = platform.uname().machine - pcbuild = os.path.join(srcdir, 'PCbuild', machine) - if os.path.exists(pcbuild): - # pyconfig.h is generated in PCbuild\amd64\ - include_dirs.append(pcbuild) - # python313.lib is generated in PCbuild\amd64\ - library_dirs.append(pcbuild) - print(f"Add PCbuild directory: {pcbuild}") + if incdirs: + print("Add incdirs:", incdirs) + include_dirs.extend(incdirs.split(os.pathsep)) + if libdirs: + print("Add libdirs:", libdirs) + library_dirs.extend(libdirs.split(os.pathsep)) # Display information to help debugging for env_name in ('CC', 'CXX', 'CFLAGS', 'CPPFLAGS', 'CXXFLAGS'): diff --git a/Lib/test/test_crossinterp.py b/Lib/test/test_crossinterp.py index 4e5362111687477..f4bf5a66ad21550 100644 --- a/Lib/test/test_crossinterp.py +++ b/Lib/test/test_crossinterp.py @@ -157,6 +157,10 @@ def ignore_byteswarning(): {}, {1: 7, 2: 8, 3: 9}, {1: [1], 2: (2,), 3: {3: 4}}, + # frozendict + frozendict(), + frozendict({1: 7, 2: 8, 3: 9}), + frozendict({1: [1], 2: (2,), 3: {3: 4}, 4: frozendict({5: 6})}), # set set(), {1, 2, 3}, diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index 2da1acfcf2989ee..c9d728e9ae2f9cb 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -5,7 +5,7 @@ c_short, c_int, c_long, c_longlong, c_byte, c_wchar, c_float, c_double, ArgumentError) -from test.support import import_helper, skip_if_sanitizer +from test.support import import_helper, skip_if_sanitizer, skip_emscripten_stack_overflow _ctypes_test = import_helper.import_module("_ctypes_test") @@ -193,6 +193,7 @@ class S8I(Structure): (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) @skip_if_sanitizer('requires deep stack', thread=True) + @skip_emscripten_stack_overflow() def test_recursive_as_param(self): class A: pass diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index 46f8ff93efa9152..6dadb7b410d7034 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,6 +1,7 @@ import sys import unittest import test.support +import ctypes from ctypes import (CDLL, PyDLL, ArgumentError, Structure, Array, Union, _Pointer, _SimpleCData, _CFuncPtr, @@ -247,6 +248,13 @@ def test_parameter_repr(self): self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") + if hasattr(ctypes, 'c_double_complex'): + self.assertRegex(repr(ctypes.c_double_complex.from_param(0)), + r"^$") + self.assertRegex(repr(ctypes.c_float_complex.from_param(0)), + r"^$") + self.assertRegex(repr(ctypes.c_longdouble_complex.from_param(0)), + r"^$") @test.support.cpython_only def test_from_param_result_refcount(self): diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 92d4851d739d47b..65ccc12625f72be 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -299,9 +299,16 @@ class X(Structure): self.assertEqual(s.first, got.first) self.assertEqual(s.second, got.second) + @unittest.skipIf(support.is_wasm32, "wasm ABI is incompatible with test expectations") def _test_issue18060(self, Vector): # Regression tests for gh-62260 + # This test passes a struct of two doubles by value to atan2(), whose C + # signature is atan2(double, double), so it only works on platforms + # where the abi of a function that takes a struct with two doubles + # matches the abi of a function that takes two doubles. The wasm32 ABI + # does not satisfy this condition and the test breaks. + # The call to atan2() should succeed if the # class fields were correctly cloned in the # subclasses. Otherwise, it will segfault. diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index d5ca7f2ca1ae658..721cde39861ce97 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -1,15 +1,17 @@ import functools import inspect import os +import select import string import sys import tempfile +import threading import unittest from unittest.mock import MagicMock from test.support import (requires, verbose, SaveSignals, cpython_only, check_disallow_instantiation, MISSING_C_DOCSTRINGS, - gc_collect) + gc_collect, SHORT_TIMEOUT) from test.support.import_helper import import_module # Optionally test curses module. This currently requires that the @@ -53,9 +55,11 @@ def wrapped(self, *args, **kwargs): term = os.environ.get('TERM') SHORT_MAX = 0x7fff -# If newterm was supported we could use it instead of initscr and not exit +# newterm() is used when available (it reports errors instead of exiting), but +# initscr() is still the fallback, and an unusable $TERM has no terminal to +# drive either way. @unittest.skipIf(not term or term == 'unknown', - "$TERM=%r, calling initscr() may cause exit" % term) + "$TERM=%r, no usable terminal" % term) @unittest.skipIf(sys.platform == "cygwin", "cygwin's curses mostly just hangs") class TestCurses(unittest.TestCase): @@ -110,7 +114,25 @@ def setUp(self): sys.stderr.flush() sys.stdout.flush() print(file=self.output, flush=True) - self.stdscr = curses.initscr() + if hasattr(curses, 'newterm'): + # Use newterm() rather than initscr(): it reports errors instead of + # exiting, and gives each test a fresh screen, which also lets + # ScreenTests run newterm()/set_term() in the same process. + try: + infd = sys.__stdin__.fileno() + except (AttributeError, ValueError, OSError): + infd = stdout_fd + self.screen = curses.newterm(term, stdout_fd, infd) + self.stdscr = self.screen.stdscr + # Close the screen after the test to break its window<->screen + # reference cycle deterministically, rather than leaving it for the + # cyclic GC to collect during a much later test (where a window's + # delwin() can fail -- an unraisable error on macOS). + self.addCleanup(self.screen.close) + self.addCleanup(setattr, self, 'screen', None) + self.addCleanup(setattr, self, 'stdscr', None) + else: + self.stdscr = curses.initscr() if self.isatty: curses.savetty() self.addCleanup(curses.endwin) @@ -119,9 +141,12 @@ def setUp(self): @requires_curses_func('filter') def test_filter(self): - # TODO: Should be called before initscr() or newterm() are called. - # TODO: nofilter() + # filter() must be called before initscr()/newterm(); it confines + # curses to a single line. Undo it with nofilter() afterwards so that + # it does not shrink the screens created by later tests. curses.filter() + if hasattr(curses, 'nofilter'): + self.addCleanup(curses.nofilter) @requires_curses_func('use_env') def test_use_env(self): @@ -252,6 +277,85 @@ def test_refresh_control(self): self.assertIs(win.is_wintouched(), syncok) self.assertIs(stdscr.is_wintouched(), syncok) + def _encodable(self, s): + # Wide characters are only supported in a locale that can encode them. + try: + s.encode(self.stdscr.encoding) + except UnicodeEncodeError: + return False + return True + + @requires_curses_window_meth('get_wch') + def test_addch_combining(self): + stdscr = self.stdscr + stdscr.move(0, 0) + # A character cell may hold a spacing char plus combining marks. + if self._encodable('e\u0301'): + stdscr.addch('e\u0301') # 'e' + COMBINING ACUTE ACCENT + if self._encodable('a\u0323\u0300'): + stdscr.addch(1, 0, 'a\u0323\u0300') # base plus two combining marks + # Too many code points to fit in a single character cell. + self.assertRaises(TypeError, stdscr.addch, 'e' + '\u0301' * 10) + # Only the first code point may be a spacing character. + self.assertRaises(ValueError, stdscr.addch, 'ab') + self.assertRaises(ValueError, stdscr.addch, 'a\u0301b') + # A lone control character is allowed (like addch(ord('\n'))), but it + # cannot be combined with other characters, as base or otherwise. + stdscr.addch('\n') + self.assertRaises(ValueError, stdscr.addch, 'a\n') + self.assertRaises(ValueError, stdscr.addch, '\n\u0301') + self.assertRaises(ValueError, stdscr.addch, '\ne\u0301') + + @requires_curses_window_meth('get_wch') + def test_addch_emoji(self): + # curses has no grapheme-cluster support: a cell holds one spacing + # character plus zero-width combining characters. A lone emoji fits, + # as does an emoji with a zero-width variation selector. + stdscr = self.stdscr + if self._encodable('\U0001f600'): + stdscr.addch(0, 0, '\U0001f600') # single emoji + if self._encodable('\u263a\ufe0f'): + stdscr.addch(1, 0, '\u263a\ufe0f') # WHITE SMILING FACE + VS-16 + # An emoji ZWJ sequence or an emoji with a modifier is more than one + # spacing character and cannot share a single cell. + self.assertRaises(ValueError, stdscr.addch, + '\U0001f44d\U0001f3fd') # thumbs up + skin tone + self.assertRaises(ValueError, stdscr.addch, + '\U0001f468\u200d\U0001f469') # man ZWJ woman + + @requires_curses_window_meth('get_wch') + def test_wide_characters(self): + # Wide and combining characters in the character-cell methods. + stdscr = self.stdscr + combining = 'e\u0301' # 'e' + COMBINING ACUTE ACCENT + vline, hline = '\u2502', '\u2500' # box-drawing vertical/horizontal + stdscr.move(0, 0) + if self._encodable(combining): + stdscr.echochar(combining) + stdscr.insch(1, 0, combining) + stdscr.bkgdset(combining) + stdscr.bkgd(combining) + if self._encodable(hline): + stdscr.hline(2, 0, hline, 5) + if self._encodable(vline): + stdscr.vline(3, 0, vline, 3) + if self._encodable(vline + hline): + stdscr.border(vline, vline, hline, hline) + stdscr.box(vline, hline) + # border() and box() cannot mix integer and wide-string characters. + self.assertRaises(TypeError, stdscr.box, vline, ord('-')) + + + @requires_curses_window_meth('in_wstr') + def test_in_wstr(self): + # The wide-character window read returns a str (instr returns bytes). + stdscr = self.stdscr + s = 'a\u00e9\u2502z' # 'a', 'e'+acute (precomposed), box vline, 'z' + stdscr.addstr(0, 0, s) + self.assertEqual(stdscr.in_wstr(0, 0, len(s)), s) + self.assertIsInstance(stdscr.instr(0, 0, len(s)), bytes) + + def test_output_character(self): stdscr = self.stdscr encoding = stdscr.encoding @@ -280,13 +384,16 @@ def test_output_character(self): stdscr.echochar('A') stdscr.echochar(b'A') stdscr.echochar(65) - with self.assertRaises((UnicodeEncodeError, OverflowError)): - # Unicode is not fully supported yet, but at least it does - # not crash. - # It is supposed to fail because either the character is - # not encodable with the current encoding, or it is encoded to - # a multibyte sequence. - stdscr.echochar('\u0114') + c = '\u0114' + try: + stdscr.echochar(c) + except UnicodeEncodeError: + # The character is not encodable with the current encoding. + self.assertRaises(UnicodeEncodeError, c.encode, encoding) + except OverflowError: + # The character is encoded to a multibyte sequence. + encoded = c.encode(encoding) + self.assertNotEqual(len(encoded), 1, repr(encoded)) stdscr.echochar('A', curses.A_BOLD) self.assertIs(stdscr.is_wintouched(), False) @@ -334,6 +441,65 @@ def test_output_string_embedded_null_chars(self): self.assertRaises(ValueError, stdscr.insstr, arg) self.assertRaises(ValueError, stdscr.insnstr, arg, 1) + def test_add_string_behavior(self): + # addstr() advances the cursor past the written text; addnstr() + # writes at most n characters. + win = curses.newwin(1, 10, 0, 0) + win.addstr(0, 0, 'abc') + self.assertEqual(win.getyx(), (0, 3)) + win.erase() + win.addnstr(0, 0, 'abcdef', 3) + self.assertEqual(win.instr(0, 0), b'abc ') + + def test_insert_string_behavior(self): + # insstr()/insnstr() insert at the cursor, shift the rest of the + # line right (losing characters off the edge), and leave the cursor + # where it was. + win = curses.newwin(1, 10, 0, 0) + win.addstr(0, 0, 'abcde') + win.move(0, 1) + win.insstr('XY') + self.assertEqual(win.getyx(), (0, 1)) # cursor did not advance + self.assertEqual(win.instr(0, 0), b'aXYbcde ') + + win.erase() + win.addstr(0, 0, 'ZZZZZ') + win.move(0, 0) + win.insnstr('abcdef', 3) # at most 3 characters + self.assertEqual(win.instr(0, 0), b'abcZZZZZ ') + + def test_insch(self): + # insch() inserts a single character at the cursor (or at y, x), + # shifting the rest of the line right. + win = curses.newwin(2, 10, 0, 0) + win.addstr(0, 0, 'abc') + win.move(0, 1) + win.insch(ord('X')) + self.assertEqual(win.instr(0, 0), b'aXbc ') + win.insch(1, 0, 'Y', curses.A_BOLD) + self.assertEqual(win.inch(1, 0), b'Y'[0] | curses.A_BOLD) + + def test_pad(self): + pad = curses.newpad(10, 20) + pad.addstr(0, 0, 'PADTEXT') + self.assertEqual(pad.instr(0, 0, 7), b'PADTEXT') + + # subpad() creates a pad within the parent pad. Cell sharing with + # the parent is implementation-defined, so write to the subpad itself. + sub = pad.subpad(3, 5, 0, 0) + self.assertEqual(sub.getmaxyx(), (3, 5)) + sub.addstr(1, 0, 'sub') + self.assertEqual(sub.instr(1, 0, 3), b'sub') + + # A pad is refreshed onto an explicit screen rectangle; the + # 6-argument form is required (and rejected for ordinary windows). + pad.refresh(0, 0, 0, 0, 4, 10) + pad.noutrefresh(0, 0, 0, 0, 4, 10) + curses.doupdate() + self.assertRaises(TypeError, pad.refresh) + win = curses.newwin(5, 5, 0, 0) + self.assertRaises(TypeError, win.refresh, 0, 0, 0, 0, 4, 4) + def test_read_from_window(self): stdscr = self.stdscr stdscr.addstr(0, 1, 'ABCD', curses.A_BOLD) @@ -350,6 +516,27 @@ def test_read_from_window(self): self.assertRaises(ValueError, stdscr.instr, -2) self.assertRaises(ValueError, stdscr.instr, 0, 2, -2) + def test_coordinate_errors(self): + # Addressing a cell outside the window raises curses.error. + win = curses.newwin(5, 10, 0, 0) + self.assertRaises(curses.error, win.move, 100, 100) + self.assertRaises(curses.error, win.move, -1, -1) + self.assertRaises(curses.error, win.addch, 100, 100, ord('x')) + self.assertRaises(curses.error, win.inch, 100, 100) + if hasattr(win, 'chgat'): # chgat() requires wchgat() + self.assertRaises(curses.error, win.chgat, 100, 0, curses.A_BOLD) + + def test_argument_errors(self): + win = curses.newwin(5, 10, 0, 0) + # A character argument must be an int, a byte or a one-element string. + self.assertRaises(TypeError, win.addch, []) + self.assertRaises(OverflowError, win.addch, 2**64) + # A string method rejects a non-string, non-bytes argument. + self.assertRaises(TypeError, win.addstr, 5) + self.assertRaises(TypeError, win.addstr) + # Wrong number of positional arguments. + self.assertRaises(TypeError, win.instr, 0, 0, 0, 0) + def test_getch(self): win = curses.newwin(5, 12, 5, 2) @@ -661,7 +848,6 @@ def test_borders_and_lines(self): self.assertEqual(win.inch(3, 1), b'a'[0]) def test_unctrl(self): - # TODO: wunctrl() self.assertEqual(curses.unctrl(b'A'), b'A') self.assertEqual(curses.unctrl('A'), b'A') self.assertEqual(curses.unctrl(65), b'A') @@ -672,6 +858,21 @@ def test_unctrl(self): self.assertRaises(TypeError, curses.unctrl, b'AB') self.assertRaises(TypeError, curses.unctrl, '') self.assertRaises(TypeError, curses.unctrl, 'AB') + + @requires_curses_func('wunctrl') + def test_wunctrl(self): + # The wide-character variant of unctrl() returns a str. + self.assertEqual(curses.wunctrl(b'A'), 'A') + self.assertEqual(curses.wunctrl('A'), 'A') + self.assertEqual(curses.wunctrl(65), 'A') + self.assertEqual(curses.wunctrl('\n'), '^J') + self.assertEqual(curses.wunctrl(10), '^J') + self.assertEqual(curses.wunctrl('é'), 'é') # printable + self.assertRaises(TypeError, curses.wunctrl, b'') + self.assertRaises(TypeError, curses.wunctrl, b'AB') + self.assertRaises(TypeError, curses.wunctrl, '') + # More than one spacing character is not a single cell. + self.assertRaises(ValueError, curses.wunctrl, 'AB') self.assertRaises(OverflowError, curses.unctrl, 2**64) def test_endwin(self): @@ -719,7 +920,7 @@ def test_misc_module_funcs(self): curses.newpad(50, 50) def test_env_queries(self): - # TODO: term_attrs(), erasewchar(), killwchar() + # TODO: term_attrs() self.assertIsInstance(curses.termname(), bytes) self.assertIsInstance(curses.longname(), bytes) self.assertIsInstance(curses.baudrate(), int) @@ -734,6 +935,24 @@ def test_env_queries(self): self.assertIsInstance(c, bytes) self.assertEqual(len(c), 1) + # The erase and kill characters are a property of the controlling + # terminal: the wide variants report ERR (raising curses.error) without + # one, while the narrow variants above return an unspecified byte. + try: + tty_fd = os.open(os.ctermid(), os.O_RDONLY) + except OSError: + tty_fd = None + if tty_fd is not None: + os.close(tty_fd) + if hasattr(curses, 'erasewchar'): + c = curses.erasewchar() + self.assertIsInstance(c, str) + self.assertEqual(len(c), 1) + if hasattr(curses, 'killwchar'): + c = curses.killwchar() + self.assertIsInstance(c, str) + self.assertEqual(len(c), 1) + def test_output_options(self): stdscr = self.stdscr @@ -809,6 +1028,75 @@ def test_input_options(self): stdscr.timeout(0) stdscr.timeout(5) + @requires_curses_window_meth('is_scrollok') + def test_state_getters(self): + stdscr = self.stdscr + # Each is_*() getter returns the value set by the matching setter. + for setter, getter in [ + ('clearok', 'is_cleared'), + ('idcok', 'is_idcok'), + ('idlok', 'is_idlok'), + ('keypad', 'is_keypad'), + ('leaveok', 'is_leaveok'), + ('nodelay', 'is_nodelay'), + ('notimeout', 'is_notimeout'), + ('scrollok', 'is_scrollok'), + ]: + getattr(stdscr, setter)(True) + self.assertIs(getattr(stdscr, getter)(), True) + getattr(stdscr, setter)(False) + self.assertIs(getattr(stdscr, getter)(), False) + if hasattr(stdscr, 'immedok'): + stdscr.immedok(True) + self.assertIs(stdscr.is_immedok(), True) + stdscr.immedok(False) + if hasattr(stdscr, 'syncok'): + stdscr.syncok(True) + self.assertIs(stdscr.is_syncok(), True) + stdscr.syncok(False) + + # getdelay() reflects timeout()/nodelay(). + stdscr.timeout(100) + self.assertEqual(stdscr.getdelay(), 100) + stdscr.nodelay(True) + self.assertEqual(stdscr.getdelay(), 0) + stdscr.timeout(-1) + self.assertEqual(stdscr.getdelay(), -1) + + # getscrreg() reflects setscrreg(). + stdscr.setscrreg(5, 10) + self.assertEqual(stdscr.getscrreg(), (5, 10)) + + # is_pad()/is_subwin()/getparent(). + self.assertIs(stdscr.is_pad(), False) + self.assertIs(stdscr.is_subwin(), False) + self.assertIsNone(stdscr.getparent()) + sub = stdscr.subwin(3, 3, 0, 0) + self.assertIs(sub.is_subwin(), True) + self.assertIs(sub.getparent(), stdscr) + pad = curses.newpad(5, 5) + self.assertIs(pad.is_pad(), True) + + @requires_curses_func('is_cbreak') + def test_global_state_getters(self): + if self.isatty: + curses.cbreak() + self.assertIs(curses.is_cbreak(), True) + curses.nocbreak() + self.assertIs(curses.is_cbreak(), False) + curses.raw() + self.assertIs(curses.is_raw(), True) + curses.noraw() + self.assertIs(curses.is_raw(), False) + curses.echo() + self.assertIs(curses.is_echo(), True) + curses.noecho() + self.assertIs(curses.is_echo(), False) + curses.nl() + self.assertIs(curses.is_nl(), True) + curses.nonl() + self.assertIs(curses.is_nl(), False) + @requires_curses_func('typeahead') def test_typeahead(self): curses.typeahead(sys.__stdin__.fileno()) @@ -819,6 +1107,10 @@ def test_prog_mode(self): self.skipTest('requires terminal') curses.def_prog_mode() curses.reset_prog_mode() + # def_shell_mode()/reset_shell_mode() are intentionally not exercised + # here: they capture and restore curses' "shell mode" terminal state, + # which is only meaningful before initscr(). Calling them mid-suite + # corrupts the modes that endwin() restores and breaks later tests. def test_beep(self): if (curses.tigetstr("bel") is not None @@ -986,6 +1278,54 @@ def test_init_pair(self): self.assertRaises(ValueError, curses.init_pair, 1, color, 0) self.assertRaises(ValueError, curses.init_pair, 1, 0, color) + @requires_curses_func('alloc_pair') + @requires_colors + def test_dynamic_color_pairs(self): + # alloc_pair()/find_pair()/free_pair() (extended-color extension). + fg = bg = curses.COLORS - 1 + pair = curses.alloc_pair(fg, bg) + self.assertGreater(pair, 0) + self.assertEqual(curses.pair_content(pair), (fg, bg)) + # The same combination of colors reuses the same pair. + self.assertEqual(curses.alloc_pair(fg, bg), pair) + self.assertEqual(curses.find_pair(fg, bg), pair) + # Once freed, the pair is no longer found. + self.assertIsNone(curses.free_pair(pair)) + self.assertEqual(curses.find_pair(fg, bg), -1) + + # Error paths. + for color in self.bad_colors2(): + self.assertRaises(ValueError, curses.alloc_pair, color, 0) + self.assertRaises(ValueError, curses.alloc_pair, 0, color) + self.assertRaises(ValueError, curses.find_pair, color, 0) + self.assertRaises(ValueError, curses.find_pair, 0, color) + for pair in self.bad_pairs(): + self.assertRaises(ValueError, curses.free_pair, pair) + # Color pair 0 is reserved and cannot be freed. + self.assertRaises(curses.error, curses.free_pair, 0) + + # Invalid number or type of arguments. + self.assertRaises(TypeError, curses.alloc_pair) + self.assertRaises(TypeError, curses.alloc_pair, 0) + self.assertRaises(TypeError, curses.alloc_pair, 0, 0, 0) + self.assertRaises(TypeError, curses.alloc_pair, 'red', 0) + self.assertRaises(TypeError, curses.alloc_pair, 0, 'red') + self.assertRaises(TypeError, curses.alloc_pair, fg=0, bg=0) + self.assertRaises(TypeError, curses.find_pair) + self.assertRaises(TypeError, curses.find_pair, 0) + self.assertRaises(TypeError, curses.find_pair, 0, 0, 0) + self.assertRaises(TypeError, curses.find_pair, 'red', 0) + self.assertRaises(TypeError, curses.find_pair, 0, 'red') + self.assertRaises(TypeError, curses.free_pair) + self.assertRaises(TypeError, curses.free_pair, 1, 2) + self.assertRaises(TypeError, curses.free_pair, 'red') + + @requires_curses_func('reset_color_pairs') + @requires_colors + def test_reset_color_pairs(self): + self.assertIsNone(curses.reset_color_pairs()) + self.assertRaises(TypeError, curses.reset_color_pairs, 0) + @requires_colors def test_color_attrs(self): for pair in 0, 1, 255: @@ -1004,6 +1344,22 @@ def test_use_default_colors(self): self.skipTest('cannot change color (use_default_colors() failed)') self.assertEqual(curses.pair_content(0), (-1, -1)) + @requires_curses_window_meth('use') + def test_use_window(self): + win = self.stdscr + self.assertEqual(win.use(lambda w, a, b: (w is win, a, b), 5, b=6), + (True, 5, 6)) + with self.assertRaises(ZeroDivisionError): + win.use(lambda w: 1 / 0) + + @unittest.skipUnless(hasattr(curses.screen, 'use'), + 'requires screen.use()') + def test_use_screen(self): + screen = self.screen + self.assertEqual( + screen.use(lambda sc, flag: (sc is screen, flag), flag=True), + (True, True)) + @requires_curses_func('assume_default_colors') @requires_colors def test_assume_default_colors(self): @@ -1031,7 +1387,8 @@ def test_keyname(self): @requires_curses_func('has_key') def test_has_key(self): - curses.has_key(13) + self.assertIsInstance(curses.has_key(13), bool) + self.assertIsInstance(curses.has_key(curses.KEY_LEFT), bool) @requires_curses_func('getmouse') def test_getmouse(self): @@ -1069,6 +1426,10 @@ def test_userptr_memory_leak(self): def test_userptr_segfault(self): w = curses.newwin(10, 10) panel = curses.panel.new_panel(w) + # set_userptr(A()) makes a panel<->userptr reference cycle (A.__del__ + # closes over panel); clean it up so the panel and its window do not + # linger until a later test collects them. + self.addCleanup(self._delete_panels, panel) class A: def __del__(self): panel.set_userptr(None) @@ -1083,6 +1444,200 @@ def test_disallow_instantiation(self): panel = curses.panel.new_panel(w) check_disallow_instantiation(self, type(panel)) + @requires_curses_func('panel') + def test_panel_stack(self): + panel = curses.panel + # new_panel() puts the panel on top of the stack, so the three + # panels end up ordered bottom -> top as p1, p2, p3. + p1 = panel.new_panel(curses.newwin(3, 6, 0, 0)) + p2 = panel.new_panel(curses.newwin(3, 6, 1, 1)) + p3 = panel.new_panel(curses.newwin(3, 6, 2, 2)) + self.addCleanup(self._delete_panels, p1, p2, p3) + + # The most recently created panel is on top. + self.assertIs(panel.top_panel(), p3) + # window() returns the wrapped window. + self.assertEqual(p2.window().getbegyx(), (1, 1)) + + # above()/below() walk the stack one step at a time. + self.assertIs(p1.above(), p2) + self.assertIs(p2.above(), p3) + self.assertIsNone(p3.above()) # nothing above the top panel + self.assertIs(p3.below(), p2) + self.assertIs(p2.below(), p1) + + # top() raises a panel to the top, bottom() lowers it to the bottom. + p1.top() + self.assertIs(panel.top_panel(), p1) + self.assertIsNone(p1.above()) + p1.bottom() + self.assertIs(panel.bottom_panel(), p1) + self.assertIsNone(p1.below()) + + # update_panels() refreshes the virtual screen from the stack. + panel.update_panels() + + @requires_curses_func('panel') + def test_panel_hide_show(self): + p = curses.panel.new_panel(curses.newwin(3, 6, 0, 0)) + self.addCleanup(self._delete_panels, p) + self.assertIs(p.hidden(), False) + p.hide() + self.assertIs(p.hidden(), True) + p.show() + self.assertIs(p.hidden(), False) + + @requires_curses_func('panel') + def test_panel_move(self): + win = curses.newwin(3, 6, 1, 2) + p = curses.panel.new_panel(win) + self.addCleanup(self._delete_panels, p) + self.assertEqual(win.getbegyx(), (1, 2)) + p.move(4, 5) + self.assertEqual(win.getbegyx(), (4, 5)) + + @requires_curses_func('panel') + def test_panel_replace(self): + win1 = curses.newwin(3, 6, 0, 0) + win2 = curses.newwin(4, 8, 1, 1) + p = curses.panel.new_panel(win1) + self.addCleanup(self._delete_panels, p) + self.assertIs(p.window(), win1) + p.replace(win2) + self.assertIs(p.window(), win2) + + @requires_curses_func('panel') + def test_panel_userptr(self): + p = curses.panel.new_panel(curses.newwin(3, 6, 0, 0)) + self.addCleanup(self._delete_panels, p) + obj = ['userptr'] + p.set_userptr(obj) + self.assertIs(p.userptr(), obj) + + def _delete_panels(self, *panels): + # Drop the panels from the global stack so they do not leak into + # later tests that inspect top_panel()/bottom_panel(). + for p in panels: + try: + p.bottom() + except curses.panel.error: + pass + del panels + gc_collect() + + def _make_textbox(self, nlines, ncols, *, insert_mode=False, stripspaces=1): + win = curses.newwin(nlines, ncols, 0, 0) + box = curses.textpad.Textbox(win, insert_mode=insert_mode) + box.stripspaces = stripspaces + return box, win + + def _type(self, box, text): + for ch in text: + box.do_command(ch if isinstance(ch, int) else ord(ch)) + + def test_textbox_gather(self): + # Typed text is read back by gather(). With stripspaces on (the + # default) gather() keeps a single trailing blank on a line and + # drops trailing empty lines. + box, win = self._make_textbox(3, 10) + self._type(box, 'Hello') + self.assertEqual(box.gather(), 'Hello \n') + + def test_textbox_gather_multiline(self): + box, win = self._make_textbox(3, 10) + self._type(box, 'ab') + box.do_command(curses.ascii.NL) # ^j -> start of next line + self._type(box, 'cd') + self.assertEqual(box.gather(), 'ab \ncd \n') + + def test_textbox_stripspaces(self): + box, win = self._make_textbox(1, 8, stripspaces=1) + self._type(box, 'hi') + self.assertEqual(box.gather(), 'hi ') + + box, win = self._make_textbox(1, 8, stripspaces=0) + self._type(box, 'hi') + self.assertEqual(box.gather(), 'hi ') + + def test_textbox_insert_mode(self): + # In insert mode a typed character shifts the rest of the line right. + box, win = self._make_textbox(1, 10, insert_mode=True) + self._type(box, 'aXc') + win.move(0, 1) + self._type(box, 'b') + self.assertEqual(box.gather(), 'abXc ') + + def test_textbox_movement(self): + box, win = self._make_textbox(3, 10) + self._type(box, 'abc') + box.do_command(curses.ascii.SOH) # ^a -> left edge + self.assertEqual(win.getyx(), (0, 0)) + box.do_command(curses.ascii.ENQ) # ^e -> end of line + self.assertEqual(win.getyx(), (0, 3)) + + def test_textbox_kill_to_eol(self): + box, win = self._make_textbox(1, 10) + self._type(box, 'abcdef') + win.move(0, 3) + box.do_command(curses.ascii.VT) # ^k -> clear to end of line + self.assertEqual(box.gather(), 'abc ') + + def test_textbox_backspace(self): + box, win = self._make_textbox(1, 10) + self._type(box, 'abc') + box.do_command(curses.ascii.BS) # ^h -> delete backward + self.assertEqual(box.gather(), 'ab ') + + def test_textbox_edit(self): + # edit() reads characters until Ctrl-G and returns the contents. + box, win = self._make_textbox(1, 10) + for ch in reversed('Hi' + chr(curses.ascii.BEL)): + curses.ungetch(ch) + self.assertEqual(box.edit(), 'Hi ') + + def test_textbox_edit_validate(self): + # The validate hook can rewrite an incoming keystroke. + box, win = self._make_textbox(1, 10) + for ch in reversed('abc' + chr(curses.ascii.BEL)): + curses.ungetch(ch) + box.edit(lambda ch: ord('X') if ch == ord('b') else ch) + self.assertEqual(box.gather(), 'aXc ') + + def test_textpad_rectangle(self): + # rectangle() draws a box with ACS line/corner characters. + win = curses.newwin(6, 12, 0, 0) + curses.textpad.rectangle(win, 0, 0, 4, 8) + chartext = curses.A_CHARTEXT + self.assertEqual(win.inch(0, 0) & chartext, + curses.ACS_ULCORNER & chartext) + self.assertEqual(win.inch(0, 8) & chartext, + curses.ACS_URCORNER & chartext) + self.assertEqual(win.inch(4, 0) & chartext, + curses.ACS_LLCORNER & chartext) + self.assertEqual(win.inch(4, 8) & chartext, + curses.ACS_LRCORNER & chartext) + self.assertEqual(win.inch(0, 1) & chartext, + curses.ACS_HLINE & chartext) + self.assertEqual(win.inch(1, 0) & chartext, + curses.ACS_VLINE & chartext) + + def test_wrapper(self): + # wrapper() sets up curses, passes the screen to the callable along + # with extra arguments, returns its result and restores the terminal. + if not self.isatty: + self.skipTest('requires terminal') + + def body(stdscr, a, b): + self.assertIsInstance(stdscr, type(self.stdscr)) + self.assertIs(curses.isendwin(), False) + return a + b + + self.assertEqual(curses.wrapper(body, 2, 3), 5) + self.assertIs(curses.isendwin(), True) + # wrapper() left the screen ended; revive it so the per-test + # endwin() cleanup does not fail with ERR. + curses.doupdate() + @requires_curses_func('is_term_resized') def test_is_term_resized(self): lines, cols = curses.LINES, curses.COLS @@ -1107,9 +1662,11 @@ def test_resize_term(self): curses.resize_term(35000, 1) with self.assertRaises(OverflowError): curses.resize_term(1, 35000) - # GH-120378: Overflow failure in resize_term() causes refresh to fail - tmp = curses.initscr() - tmp.erase() + # GH-120378: a failed resize can leave refresh broken; restore the + # original size to recover. Avoid initscr(), which would switch away + # from the shared newterm() screen and corrupt later tests. + curses.resize_term(lines, cols) + self.stdscr.erase() @requires_curses_func('resizeterm') def test_resizeterm(self): @@ -1129,9 +1686,11 @@ def test_resizeterm(self): curses.resizeterm(35000, 1) with self.assertRaises(OverflowError): curses.resizeterm(1, 35000) - # GH-120378: Overflow failure in resizeterm() causes refresh to fail - tmp = curses.initscr() - tmp.erase() + # GH-120378: a failed resize can leave refresh broken; restore the + # original size to recover. Avoid initscr(), which would switch away + # from the shared newterm() screen and corrupt later tests. + curses.resizeterm(lines, cols) + self.stdscr.erase() def test_ungetch(self): curses.ungetch(b'A') @@ -1437,5 +1996,141 @@ def test_move_down(self): self.mock_win.reset_mock() +@unittest.skipUnless(hasattr(curses, 'newterm'), 'requires curses.newterm()') +@unittest.skipIf(not term or term == 'unknown', + "$TERM=%r, newterm() may not work" % term) +@unittest.skipIf(sys.platform == "cygwin", + "cygwin's curses mostly just hangs") +class ScreenTests(unittest.TestCase): + # newterm()/set_term() mutate global curses state, but each test drives its + # own pseudo-terminal(s) and never touches the screen shared by TestCurses, + # whose setUp() makes that screen current again. So these can run in this + # process, without a real terminal and without a subprocess. + + def setUp(self): + # newterm() may install signal handlers; restore them afterwards. + self.save_signals = SaveSignals() + self.save_signals.save() + self.addCleanup(self.save_signals.restore) + + def tearDown(self): + # Leave visual mode and reclaim the test's screens while their + # pseudo-terminals are still open (make_pty() closes them later). + try: + curses.endwin() + except curses.error: + pass + gc_collect() + + @staticmethod + def _drain_pty(master, stop): + # Read and discard whatever curses writes to the screen, until asked to + # stop and nothing more is pending. poll() rather than a blocking + # read() so we can stop without closing the fd (closing it while this + # thread is blocked in read() hangs on macOS). + poller = select.poll() + poller.register(master, select.POLLIN) + while True: + if poller.poll(100): + try: + if not os.read(master, 1024): + break # EOF + except OSError: + break + elif stop.is_set(): + break + + def make_pty(self): + master, slave = os.openpty() + # Nothing reads the master end, so writing to the slave and the + # tcdrain() in endwin() can block on macOS once the pty buffer fills; + # drain it from a background thread (endwin() releases the GIL). + stop = threading.Event() + reader = threading.Thread(target=self._drain_pty, args=(master, stop), + daemon=True) + reader.start() + # Stop and join the reader before closing the fds: on macOS, closing + # either end while the reader is blocked in read() hangs. + def stop_reader(): + stop.set() + reader.join(SHORT_TIMEOUT) + self.addCleanup(os.close, master) + self.addCleanup(os.close, slave) + self.addCleanup(stop_reader) + return slave + + def test_newterm(self): + s = self.make_pty() + screen = curses.newterm('xterm', s, s) + self.assertIsInstance(screen, curses.screen) + win = screen.stdscr + self.assertIsInstance(win, curses.window) + self.assertEqual(win.getmaxyx(), (24, 80)) + win.addstr(0, 0, 'hello') + win.refresh() + + def test_newterm_file_object(self): + # type=None uses $TERM; the file arguments accept file objects too. + s = self.make_pty() + out = os.fdopen(os.dup(s), 'wb', buffering=0) + self.addCleanup(out.close) + screen = curses.newterm(None, out, s) + self.assertIsInstance(screen, curses.screen) + + def test_set_term(self): + s = self.make_pty() + s2 = self.make_pty() + a = curses.newterm('xterm', s, s) # current screen is a + b = curses.newterm('xterm', s2, s2) # current screen is b + self.assertIs(curses.set_term(a), b) # returns the previous one + self.assertIs(curses.set_term(b), a) + + def test_window_keeps_screen_alive(self): + # The standard window keeps its screen alive; dropping every other + # reference and collecting must not invalidate the window. + s = self.make_pty() + win = curses.newterm('xterm', s, s).stdscr + gc_collect() + win.addstr(0, 0, 'still alive') + win.refresh() + + def test_screen_freed(self): + # Dropping all references to a (non-current) screen and its windows + # frees it without error. + s = self.make_pty() + s2 = self.make_pty() + a = curses.newterm('xterm', s, s) + b = curses.newterm('xterm', s2, s2) # a is no longer current + del a + gc_collect() + + def test_close(self): + s = self.make_pty() + screen = curses.newterm('xterm', s, s) + win = screen.stdscr + self.assertIsInstance(win, curses.window) + screen.close() + # After close() the standard window is detached and unusable, and + # stdscr is None. No reference cycle remains. + self.assertIsNone(screen.stdscr) + self.assertRaises(curses.error, win.addstr, 0, 0, 'x') + # close() is idempotent. + screen.close() + + @unittest.skipUnless(hasattr(curses, 'new_prescr'), + 'requires curses.new_prescr()') + def test_new_prescr(self): + screen = curses.new_prescr() + self.assertIsInstance(screen, curses.screen) + self.assertIsNone(screen.stdscr) + del screen + gc_collect() + + @cpython_only + def test_disallow_instantiation(self): + # The screen type cannot be instantiated directly (bpo-43916). + check_disallow_instantiation(self, curses.screen) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index dcd6a3ef9abfab6..423247c92ce3c29 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -2792,6 +2792,55 @@ def __eq__(self, other): self.assertEqual(C(1), 5) self.assertNotEqual(C(1), 1) + def test_eq_field_by_field(self): + @dataclasses.dataclass + class Point: + x: int + y: int + + p1 = Point(1, 2) + p2 = Point(1, 2) + p3 = Point(2, 1) + self.assertEqual(p1, p2) + self.assertNotEqual(p1, p3) + + def test_eq_type_check(self): + @dataclasses.dataclass + class A: + x: int + + @dataclasses.dataclass + class B: + x: int + + a = A(1) + b = B(1) + self.assertNotEqual(a, b) + + def test_eq_custom_field(self): + class AlwaysEqual(int): + def __eq__(self, other): + return True + + @dataclasses.dataclass + class Foo: + x: AlwaysEqual + y: int + + f1 = Foo(AlwaysEqual(1), 2) + f2 = Foo(AlwaysEqual(2), 2) + self.assertEqual(f1, f2) + + def test_eq_nan_field(self): + @dataclasses.dataclass + class D: + x: float + + nan = float('nan') + d1 = D(nan) + d2 = D(nan) + self.assertNotEqual(d1, d2) + class TestOrdering(unittest.TestCase): def test_functools_total_ordering(self): @@ -5289,6 +5338,15 @@ class A: self.assertEqual(len(fs), 1) self.assertEqual(fs[0].name, 'x') + def test_makedataclass_with_qualname(self): + A = make_dataclass("A", ['a'], qualname='ClassA') + self.assertEqual(A.__qualname__, 'ClassA') + + B = make_dataclass("B", ['b'], qualname='module1.ClassB') + self.assertEqual(B.__qualname__, 'module1.ClassB') + + C = make_dataclass("C", ['c']) + self.assertEqual(C.__qualname__, 'C') class TestZeroArgumentSuperWithSlots(unittest.TestCase): def test_zero_argument_super(self): diff --git a/Lib/test/test_defaultdict.py b/Lib/test/test_defaultdict.py index a193eb10f16d178..cc78f01e3e2ebdc 100644 --- a/Lib/test/test_defaultdict.py +++ b/Lib/test/test_defaultdict.py @@ -186,6 +186,18 @@ def test_union(self): with self.assertRaises(TypeError): i |= None + # frozendict + i_fd = i | frozendict(s) + self.assertIs(type(i_fd), defaultdict) + self.assertIs(i_fd.default_factory, int) + self.assertDictEqual(i_fd, {1: "one", 2: 2, 0: "zero"}) + self.assertEqual(list(i_fd), [1, 2, 0]) + + fd_i = frozendict(s) | i + self.assertIs(type(fd_i), frozendict) + self.assertEqual(fd_i, {1: "one", 2: 2, 0: "zero"}) + self.assertEqual(list(fd_i), [0, 1, 2]) + def test_factory_conflict_with_set_value(self): key = "conflict_test" count = 0 diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 4e1a489205a6855..3c45032cda91387 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -287,6 +287,22 @@ def test_index(self): else: self.assertEqual(d.index(element, start, stop), target) + # Test stop argument + for elem in d: + index = d.index(elem) + self.assertEqual( + index, + d.index(elem, 0), + ) + self.assertEqual( + index, + d.index(elem, 0, len(d)), + ) + self.assertEqual( + index, + d.index(elem, 0, len(d) + 100), + ) + # Test large start argument d = deque(range(0, 10000, 10)) for step in range(100): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index b2f4363b23e7480..6d61c71e162f8be 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1868,6 +1868,11 @@ def test_merge(self): self.assertEqual(fd | {}, fd) self.assertEqual(frozendict() | fd, fd) + # gh-149676: Test hash(frozendict | frozendict) + a = frozendict({"a": 1}) + b = frozendict({"b": 2}) + self.assertEqual(hash(a | b), hash(frozendict({"a": 1, "b": 2}))) + def test_update(self): # test "a |= b" operator d = frozendict(x=1) @@ -1898,10 +1903,35 @@ def test_hash(self): self.assertEqual(hash(frozendict(x=1, y=2)), hash(frozendict(y=2, x=1))) + # Check that hash() computes the hash of (key, value) pairs + cases = [ + frozendict(a=False, b=True, c=True), + frozendict(a=True, b=False, c=True), + frozendict(a=True, b=True, c=False), + frozendict({False: "a", "b": True, "c": True}), + frozendict({"a": "b", False: True, True: "c"}), + ] + hashes = {hash(fd) for fd in cases} + self.assertEqual(len(hashes), len(cases)) + fd = frozendict(x=[1], y=[2]) with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): hash(fd) + @support.cpython_only + def test_hash_cpython(self): + # Check that hash(frozendict) implementation is: + # hash(frozenset(fd.items())) + for fd in ( + frozendict(), + frozendict(x=1, y=2), + frozendict(y=2, x=1), + frozendict(a=False, b=True, c=True), + frozendict.fromkeys('abc'), + ): + with self.subTest(fd=fd): + self.assertEqual(hash(fd), hash(frozenset(fd.items()))) + def test_fromkeys(self): self.assertEqual(frozendict.fromkeys('abc'), frozendict(a=None, b=None, c=None)) @@ -1909,8 +1939,11 @@ def test_fromkeys(self): # Subclass which overrides the constructor created = frozendict(x=1) class FrozenDictSubclass(frozendict): - def __new__(self): - return created + def __new__(cls, *args, **kwargs): + if args or kwargs: + return super().__new__(cls, *args, **kwargs) + else: + return created fd = FrozenDictSubclass.fromkeys("abc") self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None)) @@ -1922,6 +1955,20 @@ def __new__(self): self.assertEqual(type(fd), FrozenDictSubclass) self.assertEqual(created, frozendict(x=1)) + # Dict subclass with a constructor which returns a frozendict + # by default + class DictSubclass(dict): + def __new__(cls, *args, **kwargs): + if args or kwargs: + return super().__new__(cls, *args, **kwargs) + else: + return created + + fd = DictSubclass.fromkeys("abc") + self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None)) + self.assertEqual(type(fd), DictSubclass) + self.assertEqual(created, frozendict(x=1)) + # Subclass which doesn't override the constructor class FrozenDictSubclass2(frozendict): pass @@ -1930,16 +1977,6 @@ class FrozenDictSubclass2(frozendict): self.assertEqual(fd, frozendict(a=None, b=None, c=None)) self.assertEqual(type(fd), FrozenDictSubclass2) - # Dict subclass which overrides the constructor - class DictSubclass(dict): - def __new__(self): - return created - - fd = DictSubclass.fromkeys("abc") - self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None)) - self.assertEqual(type(fd), DictSubclass) - self.assertEqual(created, frozendict(x=1)) - def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): for fd in ( diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 771fd46e042a416..46c9b2c1d8c9fc4 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -1,5 +1,7 @@ import difflib +from test import support from test.support import findfile, force_colorized +from test.support.import_helper import ensure_lazy_imports import unittest import doctest import sys @@ -644,6 +646,12 @@ def setUpModule(): difflib.HtmlDiff._default_prefix = 0 +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("difflib", {"_colorize"}) + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(difflib)) return tests diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 8be104585814f4a..c75992761d1334f 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -15,7 +15,8 @@ import unittest from test.support import (captured_stdout, requires_debug_ranges, requires_specialization, cpython_only, - os_helper, import_helper, reset_code) + os_helper, import_helper, reset_code, + requires_jit_enabled) from test.support.bytecode_helper import BytecodeTestCase @@ -1481,6 +1482,37 @@ def f(): self.assertEqual(assem_op, assem_cache) + @cpython_only + @requires_specialization + @requires_jit_enabled + def test_show_jit(self): + def loop(n): + for i in range(n): + pass + for _ in range(10): + loop(500) + line = loop.__code__.co_firstlineno + loop_dis = f"""\ +{line} RESUME_CHECK_JIT 0 + +{line+1} LOAD_GLOBAL_BUILTIN 1 (range + NULL) + LOAD_FAST_BORROW 0 (n) + CALL_BUILTIN_CLASS 1 + GET_ITER 0 + L1: FOR_ITER_RANGE 3 (to L2) + STORE_FAST 1 (i) + +{line+2} ENTER_EXECUTOR 0 + +{line+1} L2: END_FOR + POP_ITER + LOAD_COMMON_CONSTANT 7 (None) + RETURN_VALUE +""" + got = self.get_disassembly(loop, adaptive=True, show_jit=True) + self.do_disassembly_compare(got, loop_dis) + + class DisWithFileTests(DisTests): # Run the tests again, using the file arg instead of print diff --git a/Lib/test/test_dstring.py b/Lib/test/test_dstring.py new file mode 100644 index 000000000000000..d05829d4fbaada8 --- /dev/null +++ b/Lib/test/test_dstring.py @@ -0,0 +1,264 @@ +import unittest + +from test.test_string._support import TStringBaseCase + + +_dstring_prefixes = "d db df dt dr drb drf drt".split() +_dstring_prefixes += [p.upper() for p in _dstring_prefixes] + + +def d(s): + """Helper function to evaluate d-strings.""" + if '"""' in s: + return eval(f"d'''{s}'''") + else: + return eval(f'd"""{s}"""') + +def db(s): + """Helper function to evaluate db-strings.""" + if '"""' in s: + return eval(f"db'''{s}'''") + else: + return eval(f'db"""{s}"""') + +def fd(s, globals=None): + """Helper function to evaluate fd-strings.""" + if '"""' in s: + return eval(f"fd'''{s}'''", globals=globals) + else: + return eval(f'fd"""{s}"""', globals=globals) + + + +class DStringTestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex) as cm: + eval(str) + + def test_single_quote(self): + exprs = [ + f"{p}'hello, world'" for p in _dstring_prefixes + ] + [ + f'{p}"hello, world"' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must be triple-quoted", exprs) + + def test_empty_dstring(self): + exprs = [ + f"{p}''''''" for p in _dstring_prefixes + ] + [ + f'{p}""""""' for p in _dstring_prefixes + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + for prefix in _dstring_prefixes: + expr = f"{prefix}'''\n'''" + expr2 = f'{prefix}"""\n"""' + with self.subTest(expr=expr): + v = eval(expr) + v2 = eval(expr2) + if 't' in prefix.lower(): + self.assertEqual(v.strings, ("",)) + self.assertEqual(v2.strings, ("",)) + elif 'b' in prefix.lower(): + self.assertEqual(v, b"") + self.assertEqual(v2, b"") + else: + self.assertEqual(v, "") + self.assertEqual(v2, "") + + def test_missing_newline_in_plain_and_raw_prefixes(self): + exprs = [ + 'd"""x"""', + 'dr"""x"""', + 'db"""x"""', + 'drb"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def check_dbstring(self, s, expected): + self.assertEqual(d(s), expected) + self.assertEqual(db(s), expected.encode()) + + def test_dbstring(self): + # Basic dedent - remove common leading whitespace + source = """ + hello + world + """ + self.check_dbstring(source, "hello\nworld\n") + + # closing quote on same line as last content line + source = """ + hello + world""" + self.check_dbstring(source, "hello\nworld") + + # Dedent with varying indentation + source = """ + .line1 + ...line2 + line3 + ..""".replace('.', ' ') + self.check_dbstring(source, " line1\n line2\nline3\n ") + + # Dedent with tabs + source = """ +\thello +\tworld +\t""" + self.check_dbstring(source, "hello\nworld\n") + + # Mixed spaces and tabs (using common leading whitespace) + source = """ +\t\t hello +\t\t world +\t\t """ + self.check_dbstring(source, " hello\n world\n") + + # Empty lines do not affect the calculation of common leading whitespace + source = """ + hello + + world + """ + self.check_dbstring(source, "hello\n\nworld\n") + + # Lines with only whitespace also have their indentation removed. + source = """ +....hello +.. +...... +....world +....""".replace('.', ' ') + self.check_dbstring(source, "hello\n\n \nworld\n") + + # Line continuation with backslash works as usual. + # But you cannot put a backslash right after the opening quotes. + source = r""" + Hello \ + World!\ + """ + self.check_dbstring(source, "Hello World!") + + def test_raw_dstring(self): + source = r""" + path\\to\\file + keep\\n + """ + self.assertEqual(d(source), "path\\to\\file\nkeep\\n\n") + + def test_raw_dbstring(self): + source = r""" + path\\to\\file + keep\\n + """ + self.assertEqual(db(source), b"path\\to\\file\nkeep\\n\n") + + def test_dbstring_non_ascii_error(self): + with self.assertRaisesRegex(SyntaxError, "bytes can only contain ASCII literal characters"): + eval('db"""\n \u00e9\n """') + + def test_concat_bytes_and_nonbytes_error(self): + exprs = [ + 'd"""\n x\n """ db"""\n y\n """', + 'db"""\n x\n """ d"""\n y\n """', + ] + self.assertAllRaise(SyntaxError, "cannot mix bytes and nonbytes literals", exprs) + + +class DStringFStringInteractionTestCase(unittest.TestCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test_fdstring(self): + g = {"world": 42} + + source = r""" + Hello + {world} + """ + self.assertEqual(fd(source, globals=g), "Hello\n 42\n") + + source = r""" + Hello + {world} + """ + self.assertEqual(fd(source, globals=g), " Hello\n42\n ") + + source = r""" + Hello {world} Lorum + ipsum dolor sit amet, + """ + self.assertEqual(fd(source, globals=g), "Hello 42 Lorum\nipsum dolor sit amet,\n") + + def test_missing_newline_in_f_variants(self): + exprs = [ + 'df"""x"""', + 'drf"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_fdstring_conversion_and_format(self): + g = {"x": 3.14159, "name": "Alice"} + + source = r""" + {x:.2f} {name!r} + """ + self.assertEqual(fd(source, globals=g), "3.14 'Alice'\n") + + source = r""" + {x=} + """ + self.assertEqual(fd(source, globals=g), "x=3.14159\n") + + def test_concat_with_fstring(self): + expr = 'd"""\n hello\n """ f"world"' + self.assertEqual(eval(expr), "hello\nworld") + + +class DStringTStringInteractionTestCase(unittest.TestCase, TStringBaseCase): + def assertAllRaise(self, exception_type, regex, error_strings): + for str in error_strings: + with self.subTest(str=str): + with self.assertRaisesRegex(exception_type, regex): + eval(str) + + def test_missing_newline_in_t_variants(self): + exprs = [ + 'dt"""x"""', + 'drt"""x"""', + ] + self.assertAllRaise(SyntaxError, "d-string must start with a newline", exprs) + + def test_dtstring_basic(self): + name = "Python" + t = eval('dt"""\n Hello, {name}\n """', {"name": name}) + self.assertTStringEqual(t, ("Hello, ", "\n"), [(name, "name")]) + + def test_drtstring_raw_content(self): + t = eval('drt"""\n keep\\n\n """') + self.assertTStringEqual(t, ("keep\\n\n",), ()) + + def test_concat_with_tstring_is_rejected(self): + exprs = [ + 'd"""\n x\n """ t"hello"', + 't"hello" d"""\n x\n """', + 'db"""\n x\n """ t"hello"', + 't"hello" db"""\n x\n """', + ] + self.assertAllRaise( + SyntaxError, + "cannot mix t-string literals with string or bytes literals", + exprs, + ) + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index ba2fa99707cd469..3de87fc704d43ec 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -1,6 +1,7 @@ import dis import os.path import re +import signal import subprocess import sys import sysconfig @@ -33,27 +34,54 @@ def normalize_trace_output(output): result = [ row.split("\t") for row in output.splitlines() - if row and not row.startswith('#') + if row and not row.startswith('#') and not row.startswith('@') ] result.sort(key=lambda row: int(row[0])) result = [row[1] for row in result] - return "\n".join(result) + # Normalize paths to basenames (bpftrace outputs full paths) + normalized = [] + for line in result: + # Replace full paths with just the filename + line = re.sub(r'/[^:]+/([^/:]+\.py)', r'\1', line) + normalized.append(line) + return "\n".join(normalized) except (IndexError, ValueError): raise AssertionError( "tracer produced unparsable output:\n{}".format(output) ) +USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) + +def create_process_group(*args, **kwargs): + if USE_PROCESS_GROUP: + kwargs['start_new_session'] = True + return subprocess.Popen(*args, **kwargs) + +def kill_process_group(proc): + if USE_PROCESS_GROUP: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + proc.kill() + proc.communicate() # Clean up + + class TraceBackend: EXTENSION = None COMMAND = None COMMAND_ARGS = [] def run_case(self, name, optimize_python=None): - actual_output = normalize_trace_output(self.trace_python( - script_file=abspath(name + self.EXTENSION), - python_file=abspath(name + ".py"), - optimize_python=optimize_python)) + try: + actual_output = normalize_trace_output(self.trace_python( + script_file=abspath(name + self.EXTENSION), + python_file=abspath(name + ".py"), + optimize_python=optimize_python)) + except subprocess.TimeoutExpired: + raise AssertionError(f"{self.COMMAND[0]} timed out") with open(abspath(name + self.EXTENSION + ".expected")) as f: expected_output = f.read().rstrip() @@ -66,12 +94,17 @@ def generate_trace_command(self, script_file, subcommand=None): command += ["-c", subcommand] return command - def trace(self, script_file, subcommand=None): + def trace(self, script_file, subcommand=None, *, timeout=None): command = self.generate_trace_command(script_file, subcommand) - stdout, _ = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True).communicate() + proc = create_process_group(command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + try: + stdout, _ = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + kill_process_group(proc) + raise return stdout def trace_python(self, script_file, python_file, optimize_python=None): @@ -79,12 +112,17 @@ def trace_python(self, script_file, python_file, optimize_python=None): if optimize_python: python_flags.extend(["-O"] * optimize_python) subcommand = " ".join([sys.executable] + python_flags + [python_file]) - return self.trace(script_file, subcommand) + return self.trace(script_file, subcommand, timeout=60) def assert_usable(self): try: - output = self.trace(abspath("assert_usable" + self.EXTENSION)) + output = self.trace(abspath("assert_usable" + self.EXTENSION), + timeout=10) output = output.strip() + except subprocess.TimeoutExpired: + raise unittest.SkipTest( + f"{self.COMMAND[0]} timed out during usability check" + ) except (FileNotFoundError, NotADirectoryError, PermissionError) as fnfe: output = str(fnfe) if output != "probe: success": @@ -96,6 +134,8 @@ def assert_usable(self): class DTraceBackend(TraceBackend): EXTENSION = ".d" COMMAND = ["dtrace", "-q", "-s"] + if sys.platform == "sunos5": + COMMAND.insert(2, "-Z") class SystemTapBackend(TraceBackend): @@ -103,6 +143,176 @@ class SystemTapBackend(TraceBackend): COMMAND = ["stap", "-g"] +class BPFTraceBackend(TraceBackend): + EXTENSION = ".bt" + COMMAND = ["bpftrace"] + + # Inline bpftrace programs for each test case + PROGRAMS = { + "call_stack": """ + usdt:{python}:python:function__entry {{ + printf("%lld\\tfunction__entry:%s:%s:%d\\n", + nsecs, str(arg0), str(arg1), arg2); + }} + usdt:{python}:python:function__return {{ + printf("%lld\\tfunction__return:%s:%s:%d\\n", + nsecs, str(arg0), str(arg1), arg2); + }} + """, + "gc": """ + usdt:{python}:python:function__entry {{ + if (str(arg1) == "start") {{ @tracing = 1; }} + }} + usdt:{python}:python:function__return {{ + if (str(arg1) == "start") {{ @tracing = 0; }} + }} + usdt:{python}:python:gc__start {{ + if (@tracing) {{ + printf("%lld\\tgc__start:%d\\n", nsecs, arg0); + }} + }} + usdt:{python}:python:gc__done {{ + if (@tracing) {{ + printf("%lld\\tgc__done:%lld\\n", nsecs, arg0); + }} + }} + END {{ clear(@tracing); }} + """, + } + + # Which test scripts to filter by filename (None = use @tracing flag) + FILTER_BY_FILENAME = {"call_stack": "call_stack.py"} + + @staticmethod + def _filter_probe_rows(output): + return "\n".join( + line for line in output.splitlines() + if line.partition("\t")[0].isdigit() + ) + + # Expected outputs for each test case + # Note: bpftrace captures entry/return and may have slight timing + # differences compared to SystemTap due to probe firing order + EXPECTED = { + "call_stack": """function__entry:call_stack.py::0 +function__entry:call_stack.py:start:23 +function__entry:call_stack.py:function_1:1 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__return:call_stack.py:function_1:2 +function__entry:call_stack.py:function_2:5 +function__entry:call_stack.py:function_1:1 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__return:call_stack.py:function_1:2 +function__return:call_stack.py:function_2:6 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__entry:call_stack.py:function_4:13 +function__return:call_stack.py:function_4:14 +function__entry:call_stack.py:function_5:18 +function__return:call_stack.py:function_5:21 +function__return:call_stack.py:start:28 +function__return:call_stack.py::30""", + "gc": """gc__start:0 +gc__done:0 +gc__start:1 +gc__done:0 +gc__start:2 +gc__done:0 +gc__start:2 +gc__done:1""", + } + + def run_case(self, name, optimize_python=None): + if name not in self.PROGRAMS: + raise unittest.SkipTest(f"No bpftrace program for {name}") + + python_file = abspath(name + ".py") + python_flags = [] + if optimize_python: + python_flags.extend(["-O"] * optimize_python) + + subcommand = [sys.executable] + python_flags + [python_file] + program = self.PROGRAMS[name].format(python=sys.executable) + + try: + proc = create_process_group( + ["bpftrace", "-e", program, "-c", " ".join(subcommand)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = proc.communicate(timeout=60) + except subprocess.TimeoutExpired: + kill_process_group(proc) + raise AssertionError("bpftrace timed out") + except (FileNotFoundError, PermissionError) as e: + raise unittest.SkipTest(f"bpftrace not available: {e}") + + if proc.returncode != 0: + raise AssertionError( + f"bpftrace failed with code {proc.returncode}:\n{stderr}" + ) + + stdout = self._filter_probe_rows(stdout) + + # Filter output by filename if specified (bpftrace captures everything) + if name in self.FILTER_BY_FILENAME: + filter_filename = self.FILTER_BY_FILENAME[name] + filtered_lines = [ + line for line in stdout.splitlines() + if filter_filename in line + ] + stdout = "\n".join(filtered_lines) + + actual_output = normalize_trace_output(stdout) + expected_output = self.EXPECTED[name].strip() + + return (expected_output, actual_output) + + def assert_usable(self): + # Check if bpftrace is available and can attach to USDT probes + program = f'usdt:{sys.executable}:python:function__entry {{ printf("probe: success\\n"); exit(); }}' + try: + proc = create_process_group( + ["bpftrace", "-e", program, "-c", f"{sys.executable} -c pass"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = proc.communicate(timeout=10) + except subprocess.TimeoutExpired: + kill_process_group(proc) + raise unittest.SkipTest("bpftrace timed out during usability check") + except OSError as e: + raise unittest.SkipTest(f"bpftrace not available: {e}") + + # Check for permission errors (bpftrace usually requires root) + if proc.returncode != 0: + raise unittest.SkipTest( + f"bpftrace(1) failed with code {proc.returncode}: {stderr}" + ) + + if "probe: success" not in stdout: + raise unittest.SkipTest( + f"bpftrace(1) failed: stdout={stdout!r} stderr={stderr!r}" + ) + + +class BPFTraceOutputTests(unittest.TestCase): + def test_filter_probe_rows_ignores_warnings(self): + output = """stdin:1-19: WARNING: found external warnings +HINT: include/vmlinux.h:1439:3: warning: declaration does not declare anything +4623214882928\tgc__start:0 +4623214885575\tgc__done:0 +""" + self.assertEqual( + BPFTraceBackend._filter_probe_rows(output), + "4623214882928\tgc__start:0\n4623214885575\tgc__done:0", + ) + + @unittest.skipIf(MS_WINDOWS, "Tests not compliant with trace on Windows.") class TraceTests: # unittest.TestCase options @@ -127,7 +337,8 @@ def test_function_entry_return(self): def test_verify_call_opcodes(self): """Ensure our call stack test hits all function call opcodes""" - opcodes = set(["CALL_FUNCTION", "CALL_FUNCTION_EX", "CALL_FUNCTION_KW"]) + # Modern Python uses CALL, CALL_KW, and CALL_FUNCTION_EX + opcodes = set(["CALL", "CALL_FUNCTION_EX", "CALL_KW"]) with open(abspath("call_stack.py")) as f: code_string = f.read() @@ -152,9 +363,6 @@ def get_function_instructions(funcname): def test_gc(self): self.run_case("gc") - def test_line(self): - self.run_case("line") - class DTraceNormalTests(TraceTests, unittest.TestCase): backend = DTraceBackend() @@ -175,6 +383,17 @@ class SystemTapOptimizedTests(TraceTests, unittest.TestCase): backend = SystemTapBackend() optimize_python = 2 + +class BPFTraceNormalTests(TraceTests, unittest.TestCase): + backend = BPFTraceBackend() + optimize_python = 0 + + +class BPFTraceOptimizedTests(TraceTests, unittest.TestCase): + backend = BPFTraceBackend() + optimize_python = 2 + + class CheckDtraceProbes(unittest.TestCase): @classmethod def setUpClass(cls): @@ -190,11 +409,14 @@ def setUpClass(cls): def get_readelf_version(): try: cmd = ["readelf", "--version"] + # Force the C locale to disable localization. + env = dict(os.environ, LC_ALL="C") proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, + env=env, ) with proc: version, stderr = proc.communicate() @@ -217,12 +439,36 @@ def get_readelf_version(): return int(match.group(1)), int(match.group(2)) def get_readelf_output(self): - command = ["readelf", "-n", sys.executable] + binary = sys.executable + if sysconfig.get_config_var("Py_ENABLE_SHARED"): + lib_dir = sysconfig.get_config_var("LIBDIR") + if not lib_dir or sysconfig.is_python_build(): + lib_dir = os.path.abspath(os.path.dirname(sys.executable)) + + lib_names = [] + for name in ( + sysconfig.get_config_var("INSTSONAME"), + sysconfig.get_config_var("LDLIBRARY"), + ): + if name and name not in lib_names: + lib_names.append(name) + + if lib_dir: + for name in lib_names: + libpython_path = os.path.join(lib_dir, name) + if os.path.exists(libpython_path): + binary = libpython_path + break + + command = ["readelf", "-n", binary] + # Force the C locale to disable localization. + env = dict(os.environ, LC_ALL="C") stdout, _ = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, + env=env, ).communicate() return stdout @@ -235,6 +481,8 @@ def test_check_probes(self): "Name: audit", "Name: gc__start", "Name: gc__done", + "Name: function__entry", + "Name: function__return", ] for probe_name in available_probe_names: @@ -247,8 +495,6 @@ def test_missing_probes(self): # Missing probes will be added in the future. missing_probe_names = [ - "Name: function__entry", - "Name: function__return", "Name: line", ] diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index aded44e85ee3368..9d9fe418ee4d067 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -1060,6 +1060,78 @@ def get_phrase_cfws_only_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_phrase(' (foo) ') + def test_get_phrase_adjacent_ew(self): + # "'linear-white-space' that separates a pair of adjacent + # 'encoded-word's is ignored" (rfc2047 section 6.2) + self._test_get_x(parser.get_phrase, '=?ascii?q?Joi?= \t =?ascii?q?ned?=', 'Joined', 'Joined', [], '') + + def test_get_phrase_adjacent_ew_different_encodings(self): + self._test_get_x( + parser.get_phrase, + '=?utf-8?q?B=C3=A9r?= =?iso-8859-1?q?=E9nice?=', 'Bérénice', 'Bérénice', [], '' + ) + + def test_get_phrase_adjacent_ew_encoded_spaces(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Encoded?= =?ascii?q?_spaces_?= =?ascii?q?preserved?=', + 'Encoded spaces preserved', + 'Encoded spaces preserved', + [], + '' + ) + + def test_get_phrase_adjacent_ew_comment_is_not_linear_white_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Comment?= (is not) =?ascii?q?linear-white-space?=', + 'Comment (is not) linear-white-space', + 'Comment linear-white-space', + [], + '', + comments=['is not'], + ) + + def test_get_phrase_adjacent_ew_no_error_on_defects(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Def?= =?ascii?q?ect still joins?=', + 'Defect still joins', + 'Defect still joins', + [errors.InvalidHeaderDefect], # whitespace inside encoded word + '' + ) + + def test_get_phrase_adjacent_ew_ignore_non_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?join?= for non-ew', + 'No =?join?= for non-ew', + 'No =?join?= for non-ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_ignore_invalid_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_missing_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Joi?==?ascii?q?ned?=', + 'Joined', + 'Joined', + [errors.InvalidHeaderDefect], # missing trailing whitespace + '' + ) + # get_local_part def test_get_local_part_simple(self): @@ -2387,6 +2459,22 @@ def test_get_address_rfc2047_display_name(self): self.assertEqual(address[0].token_type, 'mailbox') + def test_get_address_rfc2047_display_name_adjacent_ews(self): + address = self._test_get_x(parser.get_address, + '=?utf-8?q?B=C3=A9r?= =?utf-8?q?=C3=A9nice?= ', + 'Bérénice ', + 'Bérénice ', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].display_name, + 'Bérénice') + self.assertEqual(address[0].token_type, + 'mailbox') + def test_get_address_empty_group(self): address = self._test_get_x(parser.get_address, 'Monty Python:;', diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py index ca44f54c69b39bc..59013f087199e3a 100644 --- a/Lib/test/test_email/test_asian_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -58,6 +58,62 @@ def test_japanese_codecs(self): # TK: full decode comparison eq(str(h).encode(jcode), subject_bytes) + h = Header("Japanese") + s = '\u65e5\u672c\u8a9e' # 日本語 + h.append(s, Charset('euc-jp')) + h.append(s, Charset('iso-2022-jp')) + h.append(s, Charset('shift_jis')) + eq(h.encode(), """\ +Japanese =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?= =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?= + =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?=""") + eq(decode_header(h.encode()), + [(b'Japanese ', None), + (b'\x1b$BF|K\\8l\x1b(B\x1b$BF|K\\8l\x1b(B\x1b$BF|K\\8l\x1b(B', 'iso-2022-jp'), + ]) + + def test_chinese_codecs(self): + eq = self.ndiffAssertEqual + h = Header("Chinese") + s = '\u4e2d\u6587' # 中文 + h.append(s, Charset('gb2312')) + h.append(s, Charset('gbk')) + h.append(s, Charset('gb18030')) + h.append(s, Charset('hz')) + h.append(s, Charset('big5')) + h.append(s, Charset('big5hkscs')) + eq(h.encode(), """\ +Chinese =?gb2312?b?1tDOxA==?= =?gbk?b?1tDOxA==?= =?gb18030?b?1tDOxA==?= + =?hz-gb-2312?b?fntWUE5Efn0=?= =?big5?b?pKSk5Q==?= =?big5-hkscs?b?pKSk5Q==?=""") + eq(decode_header(h.encode()), + [(b'Chinese ', None), + (b'\xd6\xd0\xce\xc4', 'gb2312'), + (b'\xd6\xd0\xce\xc4', 'gbk'), + (b'\xd6\xd0\xce\xc4', 'gb18030'), + (b'~{VPND~}', 'hz-gb-2312'), + (b'\xa4\xa4\xa4\xe5', 'big5'), + (b'\xa4\xa4\xa4\xe5', 'big5-hkscs'), + ]) + + def test_korean_codecs(self): + eq = self.ndiffAssertEqual + h = Header("Korean") + s = '\ud55c\uad6d\uc5b4' # 한국어 + h.append(s, Charset('euc-kr')) + h.append(s, Charset('ks_c_5601-1987')) + h.append(s, Charset('cp949')) + h.append(s, Charset('iso-2022-kr')) + h.append(s, Charset('johab')) + eq(h.encode(), """\ +Korean =?euc-kr?b?x9Gxub7u?= =?ks_c_5601-1987?b?x9Gxub7uIMfRsbm+7g==?= + =?iso-2022-kr?b?GyQpQw5HUTE5Pm4P?= =?johab?b?0GWKgrTh?=""") + eq(decode_header(h.encode()), + [(b'Korean ', None), + (b'\xc7\xd1\xb1\xb9\xbe\xee', 'euc-kr'), + (b'\xc7\xd1\xb1\xb9\xbe\xee \xc7\xd1\xb1\xb9\xbe\xee', 'ks_c_5601-1987'), + (b'\x1b$)C\x0eGQ19>n\x0f', 'iso-2022-kr'), + (b'\xd0e\x8a\x82\xb4\xe1', 'johab'), + ]) + def test_payload_encoding_utf8(self): jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp') diff --git a/Lib/test/test_email/test_contentmanager.py b/Lib/test/test_email/test_contentmanager.py index dceb54f15e48f4e..3115941f8703194 100644 --- a/Lib/test/test_email/test_contentmanager.py +++ b/Lib/test/test_email/test_contentmanager.py @@ -342,6 +342,66 @@ def test_set_text_charset_latin_1(self): self.assertEqual(m.get_payload(decode=True).decode('utf-8'), content) self.assertEqual(m.get_content(), content) + def test_set_text_charset_cp949(self): + m = self._make_message() + content = "\ud55c\uad6d\uc5b4\n\uac02\n" + raw_data_manager.set_content(m, content, charset='cp949') + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="ks_c_5601-1987" + Content-Transfer-Encoding: base64 + + x9Gxub7uCoFBCg== + """)) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="ks_c_5601-1987" + Content-Transfer-Encoding: 8bit + + \ud55c\uad6d\uc5b4 + \uac02 + """).encode('ks_c_5601-1987')) + self.assertEqual(m.get_payload(decode=True), content.encode('ks_c_5601-1987')) + self.assertEqual(m.get_content(), content) + + def test_set_text_charset_shift_jis(self): + m = self._make_message() + content = "\u65e5\u672c\u8a9e\n" + raw_data_manager.set_content(m, content, charset='shift_jis') + self.assertEqual(m['Content-Type'], 'text/plain; charset="iso-2022-jp"') + self.assertEqual(m.get_payload(decode=True), content.encode('iso-2022-jp')) + self.assertEqual(m.get_content(), content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \x1b$BF|K\\8l\x1b(B + """)) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \u65e5\u672c\u8a9e + """).encode('iso-2022-jp')) + + def test_set_text_charset_euc_jp(self): + m = self._make_message() + content = "\u65e5\u672c\u8a9e\n" + raw_data_manager.set_content(m, content, charset='euc-jp') + self.assertEqual(m['Content-Type'], 'text/plain; charset="iso-2022-jp"') + self.assertEqual(m.get_payload(decode=True), content.encode('iso-2022-jp')) + self.assertEqual(m.get_content(), content) + self.assertEqual(str(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \x1b$BF|K\\8l\x1b(B + """)) + self.assertEqual(bytes(m), textwrap.dedent("""\ + Content-Type: text/plain; charset="iso-2022-jp" + Content-Transfer-Encoding: 7bit + + \u65e5\u672c\u8a9e + """).encode('iso-2022-jp')) + def test_set_text_plain_long_line_heuristics(self): m = self._make_message() content = ("Simple but long message that is over 78 characters" diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 7778566492d8f44..19555d87085e176 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -4970,6 +4970,128 @@ def tearDown(self): except KeyError: pass + def test_attributes(self): + from email import charset + c = Charset() + self.assertEqual(c.input_charset, 'us-ascii') + self.assertEqual(c.header_encoding, None) + self.assertEqual(c.body_encoding, None) + self.assertEqual(c.output_charset, 'us-ascii') + self.assertEqual(c.input_codec, None) + self.assertEqual(c.output_codec, None) + + c = Charset('us-ascii') + self.assertEqual(c.input_charset, 'us-ascii') + self.assertEqual(c.header_encoding, None) + self.assertEqual(c.body_encoding, None) + self.assertEqual(c.output_charset, 'us-ascii') + self.assertEqual(c.input_codec, None) + self.assertEqual(c.output_codec, None) + + c = Charset('utf8') + self.assertEqual(c.input_charset, 'utf-8') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'utf-8') + self.assertEqual(c.input_codec, 'utf-8') + self.assertEqual(c.output_codec, 'utf-8') + + c = Charset('latin1') + self.assertEqual(c.input_charset, 'iso-8859-1') + self.assertEqual(c.header_encoding, charset.QP) + self.assertEqual(c.body_encoding, charset.QP) + self.assertEqual(c.output_charset, 'iso-8859-1') + self.assertEqual(c.input_codec, 'iso-8859-1') + self.assertEqual(c.output_codec, 'iso-8859-1') + + c = Charset('latin9') + self.assertEqual(c.input_charset, 'iso-8859-15') + self.assertEqual(c.header_encoding, charset.QP) + self.assertEqual(c.body_encoding, charset.QP) + self.assertEqual(c.output_charset, 'iso-8859-15') + self.assertEqual(c.input_codec, 'iso-8859-15') + self.assertEqual(c.output_codec, 'iso-8859-15') + + c = Charset('cyrillic') + self.assertEqual(c.input_charset, 'iso-8859-5') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'iso-8859-5') + self.assertEqual(c.input_codec, 'iso-8859-5') + self.assertEqual(c.output_codec, 'iso-8859-5') + + c = Charset('cp1251') + self.assertEqual(c.input_charset, 'windows-1251') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'windows-1251') + self.assertEqual(c.input_codec, 'windows-1251') + self.assertEqual(c.output_codec, 'windows-1251') + + c = Charset('cp1252') + self.assertEqual(c.input_charset, 'windows-1252') + self.assertEqual(c.header_encoding, charset.QP) + self.assertEqual(c.body_encoding, charset.QP) + self.assertEqual(c.output_charset, 'windows-1252') + self.assertEqual(c.input_codec, 'windows-1252') + self.assertEqual(c.output_codec, 'windows-1252') + + c = Charset('eucjp') + self.assertEqual(c.input_charset, 'euc-jp') + self.assertEqual(c.header_encoding, charset.BASE64) + self.assertEqual(c.body_encoding, None) + self.assertEqual(c.output_charset, 'iso-2022-jp') + self.assertEqual(c.input_codec, 'euc-jp') + self.assertEqual(c.output_codec, 'iso-2022-jp') + + c = Charset('cp949') + self.assertEqual(c.input_charset, 'ks_c_5601-1987') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'ks_c_5601-1987') + self.assertEqual(c.input_codec, 'ks_c_5601-1987') + self.assertEqual(c.output_codec, 'ks_c_5601-1987') + + c = Charset('gb2312') + self.assertEqual(c.input_charset, 'gb2312') + self.assertEqual(c.header_encoding, charset.BASE64) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'gb2312') + self.assertEqual(c.input_codec, 'gb2312') + self.assertEqual(c.output_codec, 'gb2312') + + c = Charset('big5') + self.assertEqual(c.input_charset, 'big5') + self.assertEqual(c.header_encoding, charset.BASE64) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'big5') + self.assertEqual(c.input_codec, 'big5') + self.assertEqual(c.output_codec, 'big5') + + def test_user_charsets(self): + from email import charset + c = Charset('fake0') + self.assertEqual(c.input_charset, 'fake0') + self.assertEqual(c.header_encoding, charset.SHORTEST) + self.assertEqual(c.body_encoding, charset.BASE64) + self.assertEqual(c.output_charset, 'fake0') + self.assertEqual(c.input_codec, 'fake0') + self.assertEqual(c.output_codec, 'fake0') + + charset.add_alias('fake1', 'mime-fake') + charset.add_alias('output-mime-fake', 'output-mime-fake-alias') + charset.add_codec('mime-fake', 'fakecodec') + charset.add_codec('output-mime-fake-alias', 'outputfakecodec') + charset.add_charset('mime-fake', charset.QP, None, 'output-mime-fake') + + c = Charset('fake1') + self.assertEqual(c.input_charset, 'mime-fake') + self.assertEqual(c.header_encoding, charset.QP) + self.assertEqual(c.body_encoding, None) + self.assertEqual(c.output_charset, 'output-mime-fake-alias') + self.assertEqual(c.input_codec, 'fakecodec') + self.assertEqual(c.output_codec, 'outputfakecodec') + def test_codec_encodeable(self): eq = self.assertEqual # Make sure us-ascii = no Unicode conversion @@ -4995,15 +5117,8 @@ def test_body_encode(self): # Try the convert argument, where input codec != output codec c = Charset('euc-jp') # With apologies to Tokio Kikuchi ;) - # XXX FIXME -## try: -## eq('\x1b$B5FCO;~IW\x1b(B', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) -## eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) -## except LookupError: -## # We probably don't have the Japanese codecs installed -## pass + eq('\x1b$B5FCO;~IW\x1b(B', + c.body_encode('\u83ca\u5730\u6642\u592b')) # Testing SF bug #625509, which we have to fake, since there are no # built-in encodings where the header encoding is QP but the body # encoding is not. @@ -5017,6 +5132,11 @@ def test_unicode_charset_name(self): self.assertEqual(str(charset), 'us-ascii') self.assertRaises(errors.CharsetError, Charset, 'asc\xffii') + def test_bytes_charset_name(self): + charset = Charset(b'us-ascii') + self.assertEqual(str(charset), 'us-ascii') + self.assertRaises(errors.CharsetError, Charset, b'asc\xffii') + # Test multilingual MIME headers. diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1087cbd0836fd86..2d1533c46b98f33 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -74,6 +74,27 @@ def debug_build(program): return name.casefold().endswith("_d".casefold()) +def getpath_which(program_name): + if sys.platform != 'cygwin': + return shutil.which(program_name) + + # shutil.which() checks for os.access(fn, os.F_OK | os.X_OK), whereas + # getpath.isxfile() doesn't. The difference matters on Cygwin. + import stat + def isxfile(fn): + try: + st = os.stat(fn) + except OSError: + return False + return stat.S_ISREG(st.st_mode) + + for p in os.environ['PATH'].split(':'): + p = os.path.join(p, program_name) + if isxfile(p): + return p + return None + + def remove_python_envvars(): env = dict(os.environ) # Remove PYTHON* environment variables to get deterministic environment @@ -92,6 +113,8 @@ def setUp(self): exename += ext exepath = builddir else: + if sys.platform == 'cygwin': + exename += '.exe' exepath = os.path.join(builddir, 'Programs') self.test_exe = exe = os.path.join(exepath, exename) if not os.path.exists(exe): @@ -328,6 +351,8 @@ def test_pre_initialization_api(self): expected_path = self.test_exe else: expected_path = os.path.join(os.getcwd(), "_testembed") + if sys.platform == 'cygwin': + expected_path += '.exe' expected_output = f"sys.executable: {expected_path}\n" self.assertIn(expected_output, out) self.assertEqual(err, '') @@ -872,12 +897,16 @@ def get_expected_config(self, expected_preconfig, expected, default_executable = os.path.abspath(expected['program_name']) else: default_executable = os.path.join(os.getcwd(), '_testembed') + if sys.platform == 'cygwin': + default_executable += '.exe' if expected['executable'] is self.GET_DEFAULT_CONFIG: expected['executable'] = default_executable if expected['base_executable'] is self.GET_DEFAULT_CONFIG: expected['base_executable'] = default_executable if expected['program_name'] is self.GET_DEFAULT_CONFIG: expected['program_name'] = './_testembed' + if sys.platform == 'cygwin': + expected['program_name'] += '.exe' config = configs['config'] for key, value in expected.items(): @@ -1370,7 +1399,7 @@ def default_program_name(self, config): if MACOS: executable = self.test_exe else: - executable = shutil.which(program_name) or '' + executable = getpath_which(program_name) or '' config.update({ 'program_name': program_name, 'base_executable': executable, @@ -1469,6 +1498,13 @@ def tmpdir_with_python(self, subdir=None): shutil.copystat(self.test_exe, exec_copy) self.test_exe = exec_copy + if sys.platform == "cygwin": + # Copy libpython DLL + exe_path = os.path.dirname(sys.executable) + libpython_dll = sysconfig.get_config_var('DLLLIBRARY') + shutil.copy2(os.path.join(exe_path, libpython_dll), + os.path.join(tmpdir, libpython_dll)) + yield tmpdir def test_init_setpythonhome(self): @@ -1993,10 +2029,21 @@ def test_audit_run_stdin(self): def test_get_incomplete_frame(self): self.run_embedded_interpreter("test_get_incomplete_frame") - def test_gilstate_after_finalization(self): self.run_embedded_interpreter("test_gilstate_after_finalization") + def test_thread_state_ensure(self): + self.run_embedded_interpreter("test_thread_state_ensure") + + def test_main_interpreter_view(self): + self.run_embedded_interpreter("test_main_interpreter_view") + + def test_thread_state_ensure_from_view(self): + self.run_embedded_interpreter("test_thread_state_ensure_from_view") + + def test_concurrent_finalization_stress(self): + self.run_embedded_interpreter("test_concurrent_finalization_stress") + class MiscTests(EmbeddingTestsMixin, unittest.TestCase): def test_unicode_id_init(self): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 779457119e8f0ea..b05eab43bd9eff1 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -5021,8 +5021,8 @@ class Color(enum.Enum) | __members__ | Returns a mapping of member name->value. | - | This mapping lists all enum members, including aliases. Note that this - | is a read-only view of the internal mapping.""" + | This mapping lists all enum members, including aliases. Note that + | this is a read-only view of the internal mapping.""" expected_help_output_without_docs = """\ Help on class Color in module %s: @@ -5105,6 +5105,8 @@ def test_inspect_getmembers(self): ('__qualname__', 'TestStdLib.Color'), ('__init_subclass__', getattr(self.Color, '__init_subclass__')), ('__iter__', self.Color.__iter__), + ('_missing_', self.Color._missing_), + ('_generate_next_value_', self.Color._generate_next_value_), )) result = dict(inspect.getmembers(self.Color)) self.assertEqual(set(values.keys()), set(result.keys())) @@ -5147,6 +5149,10 @@ def test_inspect_classify_class_attrs(self): defining_class=self.Color, object='Color'), Attribute(name='__qualname__', kind='data', defining_class=self.Color, object='TestStdLib.Color'), + Attribute(name='_missing_', kind='class method', + defining_class=Enum, object=Enum.__dict__['_missing_']), + Attribute(name='_generate_next_value_', kind='static method', + defining_class=self.Color, object=self.Color.__dict__['_generate_next_value_']), Attribute(name='YELLOW', kind='data', defining_class=self.Color, object=self.Color.YELLOW), Attribute(name='MAGENTA', kind='data', @@ -5178,11 +5184,13 @@ def test_inspect_classify_class_attrs(self): # __doc__ is too big to check exactly, so treat the same as __init_subclass__ for name in ('name','kind','defining_class'): if getattr(v, name) != getattr(r, name): - print('\n%s\n%s\n%s\n%s\n' % ('=' * 75, r, v, '=' * 75), sep='') + print('\n%s\nexpected: %s\nactual: %s\n%s\n' % ('=' * 75, r, v, '=' * 75), sep='') failed = True + # breakpoint() elif r != v: - print('\n%s\n%s\n%s\n%s\n' % ('=' * 75, r, v, '=' * 75), sep='') + print('\n%s\nexpected: %s\nactual: %s\n%s\n' % ('=' * 75, r, v, '=' * 75), sep='') failed = True + # breakpoint() if failed: self.fail("result does not equal expected, see print above") @@ -5537,6 +5545,7 @@ def enum_dir(cls): '__class__', '__contains__', '__doc__', '__getitem__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__', + '_generate_next_value_', '_missing_', ] + members ) @@ -5552,7 +5561,8 @@ def enum_dir(cls): def member_dir(member): if member.__class__._member_type_ is object: - allowed = set(['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'value']) + allowed = set(['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'value', + '_generate_next_value_', '_missing_', '_add_alias_', '_add_value_alias_']) else: allowed = set(dir(member)) for cls in member.__class__.mro(): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 3f5fcb29b442dec..df7a04273b9b41c 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1714,6 +1714,20 @@ def inner(): gc_collect() # For PyPy or other GCs. self.assertEqual(wr(), None) + def test_oserror_reinit_leak(self): + # gh-150988: Check for memory leak when re-initializing OSError. + # Previously, setting OSError attributes in a subclass + # before calling super().__init__() leaked memory. + class LeakingOSError(OSError): + def __init__(self, code, message, filename, filename2): + self.strerror = message + self.filename = filename + self.filename2 = filename2 + super().__init__(code, message, filename, None, filename2) + + exc = LeakingOSError(1, "some message", "filename.py", "filename2.py") + exc.__init__(2, "another message", "filename3.py", "filename4.py") + def test_errno_ENOTDIR(self): # Issue #12802: "not a directory" errors are ENOTDIR even on Windows with self.assertRaises(OSError) as cm: diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 401136de8de6661..6b1529aa173f01c 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -1437,6 +1437,160 @@ def matches_awaited_by_pattern(task): finally: _cleanup_sockets(client_socket, server_socket) + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_global_awaited_by_from_non_main_thread(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio + import socket + import threading + import time + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + async def worker_main(): + task = asyncio.create_task( + asyncio.sleep(10_000), + name="worker task", + ) + await asyncio.sleep(0) + sock.sendall(f"ready:{{threading.get_native_id()}}\\n".encode()) + await task + + def run_worker_loop(): + asyncio.run(worker_main()) + + threading.Thread( + target=run_worker_loop, + name="async-worker", + daemon=True, + ).start() + time.sleep(10_000) + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + response = _wait_for_signal(client_socket, b"ready:") + worker_thread_id = int( + response.split(b"ready:", 1)[1].splitlines()[0] + ) + + for _ in busy_retry(SHORT_TIMEOUT): + all_awaited_by = get_all_awaited_by(p.pid) + if any( + task.task_name == "worker task" + for info in all_awaited_by + if info.thread_id == worker_thread_id + for task in info.awaited_by + ): + break + else: + self.fail( + "get_all_awaited_by() did not report " + "the asyncio task from the non-main thread" + ) + finally: + _cleanup_sockets(client_socket, server_socket) + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_async_remote_stack_trace_from_non_main_thread(self): + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import asyncio + import socket + import threading + import time + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def blocking_call(): + sock.sendall(f"ready:{{threading.get_native_id()}}\\n".encode()) + time.sleep(10_000) + + async def worker_task(): + await asyncio.sleep(0) + blocking_call() + + async def worker_main(): + task = asyncio.create_task( + worker_task(), + name="worker task", + ) + await task + + def run_worker_loop(): + asyncio.run(worker_main()) + + threading.Thread( + target=run_worker_loop, + name="async-worker", + daemon=True, + ).start() + time.sleep(10_000) + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + server_socket = _create_server_socket(port) + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + + try: + with _managed_subprocess([sys.executable, script_name]) as p: + client_socket, _ = server_socket.accept() + server_socket.close() + server_socket = None + + response = _wait_for_signal(client_socket, b"ready:") + worker_thread_id = int( + response.split(b"ready:", 1)[1].splitlines()[0] + ) + + for _ in busy_retry(SHORT_TIMEOUT): + stack_trace = get_async_stack_trace(p.pid) + if any( + task.task_name == "worker task" + for info in stack_trace + if info.thread_id == worker_thread_id + for task in info.awaited_by + ): + break + else: + self.fail( + "get_async_stack_trace() did not report " + "the running asyncio task from the non-main thread" + ) + finally: + _cleanup_sockets(client_socket, server_socket) + @skip_if_not_supported @unittest.skipIf( sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, @@ -3613,6 +3767,13 @@ def test_get_stats(self): "frames_read_from_cache", "frames_read_from_memory", "frame_cache_hit_rate", + "batched_read_attempts", + "batched_read_successes", + "batched_read_misses", + "batched_read_segments_requested", + "batched_read_segments_completed", + "batched_read_success_rate", + "batched_read_segment_completion_rate", ] for key in expected_keys: self.assertIn(key, stats) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 11df59f2346f316..5a493a4fd956802 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -33,6 +33,11 @@ CURRENT_THREAD_HEADER = fr'{CURRENT_THREAD_ID} \(most recent call first\):' +def skip_if_sanitizer_signal(signame): + return support.skip_if_sanitizer(f"TSAN/UBSan itercepts {signame}", + thread=True, ub=True) + + def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header regex += ' File "", line %s in func\n' % lineno1 @@ -224,7 +229,7 @@ def test_fatal_error_c_thread(self): func='faulthandler_fatal_error_thread', py_fatal_error=True) - @support.skip_if_sanitizer("TSAN itercepts SIGABRT", thread=True) + @skip_if_sanitizer_signal("SIGABRT") def test_sigabrt(self): self.check_fatal_error(""" import faulthandler @@ -236,7 +241,7 @@ def test_sigabrt(self): @unittest.skipIf(sys.platform == 'win32', "SIGFPE cannot be caught on Windows") - @support.skip_if_sanitizer("TSAN itercepts SIGFPE", thread=True) + @skip_if_sanitizer_signal("SIGFPE") def test_sigfpe(self): self.check_fatal_error(""" import faulthandler @@ -248,7 +253,7 @@ def test_sigfpe(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGBUS'), 'need signal.SIGBUS') - @support.skip_if_sanitizer("TSAN itercepts SIGBUS", thread=True) + @skip_if_sanitizer_signal("SIGBUS") @skip_segfault_on_android def test_sigbus(self): self.check_fatal_error(""" @@ -263,7 +268,7 @@ def test_sigbus(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGILL'), 'need signal.SIGILL') - @support.skip_if_sanitizer("TSAN itercepts SIGILL", thread=True) + @skip_if_sanitizer_signal("SIGILL") @skip_segfault_on_android def test_sigill(self): self.check_fatal_error(""" diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 222b69a6d250cd3..f20a9d407670a9c 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -151,6 +151,7 @@ def test_fcntl_64_bit(self): except OSError as exc: if exc.errno == errno.EINVAL: self.skipTest("F_NOTIFY not available by this environment") + raise fcntl.fcntl(fd, cmd, flags) finally: os.close(fd) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index 00f1ab44b0a8fa8..5d322cb444cfb68 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -68,33 +68,20 @@ def testcommon(formatstr, args, output=None, limit=None, overflowok=False): testformat(b_format, args, b_output, limit, overflowok) testformat(ba_format, args, ba_output, limit, overflowok) -def test_exc(formatstr, args, exception, excmsg): - try: - testformat(formatstr, args) - except exception as exc: - if str(exc) == excmsg: - if verbose: - print("yes") - else: - if verbose: print('no') - print('Unexpected ', exception, ':', repr(str(exc))) - raise - except: - if verbose: print('no') - print('Unexpected exception') - raise - else: - raise TestFailed('did not get expected exception: %s' % excmsg) - -def test_exc_common(formatstr, args, exception, excmsg): - # test str and bytes - test_exc(formatstr, args, exception, excmsg) - if isinstance(args, dict): - args = {k.encode('ascii'): v for k, v in args.items()} - test_exc(formatstr.encode('ascii'), args, exception, excmsg) class FormatTest(unittest.TestCase): + def check_exc(self, formatstr, args, exception, excmsg): + with self.assertRaisesRegex(exception, re.escape(excmsg)): + testformat(formatstr, args) + + def check_exc_common(self, formatstr, args, exception, excmsg): + # test str and bytes + self.check_exc(formatstr, args, exception, excmsg) + if isinstance(args, dict): + args = {k.encode('ascii'): v for k, v in args.items()} + self.check_exc(formatstr.encode('ascii'), args, exception, excmsg) + def test_common_format(self): # test the format identifiers that work the same across # str, bytes, and bytearrays (integer, float, oct, hex) @@ -271,6 +258,7 @@ def test_common_format(self): if verbose: print('Testing exceptions') + test_exc_common = self.check_exc_common test_exc_common('abc %', (), ValueError, "stray % at position 4") test_exc_common('abc % %s', 1, ValueError, "stray % at position 4 or unexpected format character '%' at position 6") @@ -365,6 +353,7 @@ def test_str_format(self): # Test exception for unknown format characters, etc. if verbose: print('Testing exceptions') + test_exc = self.check_exc test_exc('abc %b', 1, ValueError, "unsupported format %b at position 4") test_exc("abc %\nd", 1, ValueError, @@ -468,6 +457,7 @@ def __bytes__(self): # Test exception for unknown format characters, etc. if verbose: print('Testing exceptions') + test_exc = self.check_exc test_exc(b"abc %\nd", 1, ValueError, "stray % at position 4 or unexpected format character with code 0x0a at position 5") test_exc(b"abc %'d", 1, ValueError, diff --git a/Lib/test/test_free_threading/test_collections.py b/Lib/test/test_free_threading/test_collections.py index 3a413ccf396d4ba..849b0480e232fc2 100644 --- a/Lib/test/test_free_threading/test_collections.py +++ b/Lib/test/test_free_threading/test_collections.py @@ -24,6 +24,30 @@ def copy_loop(): threading_helper.run_concurrently([mutate, copy_loop]) + def test_index_race_in_ac(self): + # gh-150750: There was a c_default specified as `Py_SIZE(self)`, + # it was used without a critical section. + + d = deque(range(100)) + + def index(): + for _ in range(10000): + try: + d.index(50) + except ValueError: + pass + + def mutate(): + for _ in range(10000): + d.append(0) + d.clear() + d.extend(range(100)) + d.appendleft(-1) + + threading_helper.run_concurrently( + [index, *[mutate for _ in range(3)]], + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 55272a00c3ad501..ad23290a92ab345 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -268,6 +268,52 @@ def watcher(): finally: _testcapi.clear_dict_watcher(wid) + def test_racing_split_dict_clear_and_lookup(self): + class C: + pass + + keys = [f"a{i}" for i in range(16)] + + def make_split_nonembedded(): + inst = C() + for key in keys: + setattr(inst, key, keys.index(key)) + # dict.copy() of a split instance dict yields a split table + # with non-embedded values + return inst.__dict__.copy() + + d = make_split_nonembedded() + + def clearer(): + for _ in range(1000): + d.clear() + d.update(make_split_nonembedded()) + + def reader(): + for _ in range(1000): + for k in keys: + d.get(k) + + threading_helper.run_concurrently([clearer, reader, reader]) + + def test_racing_embedded_values_clear_and_lookup(self): + class C: + pass + + obj = C() + def writer(): + for _ in range(1000): + obj.x = 1 + obj.y = 2 + obj.z = 3 + obj.__dict__.clear() + + def reader(): + for _ in range(1000): + obj.__dict__.get('x') + + threading_helper.run_concurrently([writer, reader, reader]) + def test_racing_dict_update_and_method_lookup(self): # gh-144295: test race between dict modifications and method lookups. # Uses BytesIO because the race requires a type without Py_TPFLAGS_INLINE_VALUES diff --git a/Lib/test/test_free_threading/test_dict_watcher.py b/Lib/test/test_free_threading/test_dict_watcher.py new file mode 100644 index 000000000000000..6a6843f9344f640 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict_watcher.py @@ -0,0 +1,89 @@ +import unittest + +from test.support import import_helper, threading_helper + +_testcapi = import_helper.import_module("_testcapi") + +ITERS = 100 +NTHREADS = 20 + + +@threading_helper.requires_working_threading() +class TestDictWatcherThreadSafety(unittest.TestCase): + # Watcher kinds from _testcapi + EVENTS = 0 # appends dict events as strings to global event list + + def test_concurrent_add_clear_watchers(self): + """Race AddWatcher and ClearWatcher from multiple threads. + + Uses more threads than available watcher slots (5 user slots out + of DICT_MAX_WATCHERS=8). + """ + results = [] + + def worker(): + for _ in range(ITERS): + try: + wid = _testcapi.add_dict_watcher(self.EVENTS) + except RuntimeError: + continue # All slots taken + self.assertGreaterEqual(wid, 0) + results.append(wid) + _testcapi.clear_dict_watcher(wid) + + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify at least some watchers were successfully added + self.assertGreater(len(results), 0) + + def test_concurrent_watch_unwatch(self): + """Race Watch and Unwatch on the same dict from multiple threads.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + dicts = [{} for _ in range(10)] + + def worker(): + for _ in range(ITERS): + for d in dicts: + _testcapi.watch_dict(wid, d) + for d in dicts: + _testcapi.unwatch_dict(wid, d) + + try: + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify watching still works after concurrent watch/unwatch + _testcapi.watch_dict(wid, dicts[0]) + dicts[0]["key"] = "value" + events = _testcapi.get_dict_watcher_events() + self.assertIn("new:key:value", events) + finally: + _testcapi.clear_dict_watcher(wid) + + def test_concurrent_modify_watched_dict(self): + """Race dict mutations (triggering callbacks) with watch/unwatch.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + d = {} + _testcapi.watch_dict(wid, d) + + def mutator(): + for i in range(ITERS): + d[f"key_{i}"] = i + d.pop(f"key_{i}", None) + + def toggler(): + for i in range(ITERS): + _testcapi.watch_dict(wid, d) + d[f"toggler_{i}"] = i + _testcapi.unwatch_dict(wid, d) + + workers = [mutator, toggler] * (NTHREADS // 2) + try: + threading_helper.run_concurrently(workers) + events = _testcapi.get_dict_watcher_events() + self.assertGreater(len(events), 0) + finally: + _testcapi.clear_dict_watcher(wid) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_exceptions.py b/Lib/test/test_free_threading/test_exceptions.py new file mode 100644 index 000000000000000..61146f29c7868e5 --- /dev/null +++ b/Lib/test/test_free_threading/test_exceptions.py @@ -0,0 +1,16 @@ +import unittest +import copy + +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) +class ExceptionTests(unittest.TestCase): + def test_setstate_data_race(self): + E = Exception() + + def func(): + for i in range(100): + setattr(E, 'x', i) + copy.copy(E) + + threading_helper.run_concurrently(func, nthreads=4) diff --git a/Lib/test/test_free_threading/test_gc.py b/Lib/test/test_free_threading/test_gc.py index 8b45b6e2150c288..399010234509408 100644 --- a/Lib/test/test_free_threading/test_gc.py +++ b/Lib/test/test_free_threading/test_gc.py @@ -94,6 +94,65 @@ def evil(): thread.start() thread.join() + def test_set_threshold(self): + # GH-148613: Setting the GC threshold from another thread could cause a + # race between the `gc_should_collect` and `gc_set_threshold` functions. + NUM_THREADS = 8 + NUM_ITERS = 100_000 + barrier = threading.Barrier(NUM_THREADS) + + class CyclicReference: + def __init__(self): + self.r = self + + def allocator(): + barrier.wait() + for _ in range(NUM_ITERS): + CyclicReference() + + def setter(): + barrier.wait() + for i in range(NUM_ITERS): + gc.set_threshold(100 + (i % 100), 10 + (i % 10), 10 + (i % 10)) + + current_threshold = gc.get_threshold() + try: + threads = [Thread(target=allocator) for _ in range(NUM_THREADS - 1)] + threads.append(Thread(target=setter)) + with threading_helper.start_threads(threads): + pass + finally: + gc.set_threshold(*current_threshold) + + def test_get_count(self): + class CyclicReference: + def __init__(self): + self.ref = self + + NUM_ALLOCATORS = 7 + NUM_READERS = 1 + NUM_THREADS = NUM_ALLOCATORS + NUM_READERS + NUM_ITERS = 1000 + + barrier = threading.Barrier(NUM_THREADS) + + def allocator(): + barrier.wait() + for _ in range(NUM_ITERS): + CyclicReference() + + + def reader(): + barrier.wait() + for _ in range(NUM_ITERS): + gc.get_count() + + threads = [Thread(target=allocator) for _ in range(NUM_ALLOCATORS)] + threads.extend(Thread(target=reader) for _ in range(NUM_READERS)) + + with threading_helper.start_threads(threads): + pass + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index a51ad0cf83a0065..44d3e9ccfdd14e0 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -12,7 +12,7 @@ NUMITEMS = 1000 NUMTHREADS = 2 else: - NUMITEMS = 100000 + NUMITEMS = 5000 NUMTHREADS = 5 NUMMUTATORS = 2 diff --git a/Lib/test/test_free_threading/test_pickle.py b/Lib/test/test_free_threading/test_pickle.py new file mode 100644 index 000000000000000..45ea1bf5f26465a --- /dev/null +++ b/Lib/test/test_free_threading/test_pickle.py @@ -0,0 +1,78 @@ +import pickle +import threading +import unittest + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestPickleFreeThreading(unittest.TestCase): + + def test_pickle_dumps_with_concurrent_dict_mutation(self): + # gh-146452: Pickling a dict while another thread mutates it + # used to segfault. batch_dict_exact() iterated dict items via + # PyDict_Next() which returns borrowed references, and a + # concurrent pop/replace could free the value before Py_INCREF + # got to it. + shared = {str(i): list(range(20)) for i in range(50)} + + def dumper(): + for _ in range(1000): + try: + pickle.dumps(shared) + except RuntimeError: + # "dictionary changed size during iteration" is expected + pass + + def mutator(): + for j in range(1000): + key = str(j % 50) + shared[key] = list(range(j % 20)) + if j % 10 == 0: + shared.pop(key, None) + shared[key] = [j] + + threads = [] + for _ in range(10): + threads.append(threading.Thread(target=dumper)) + threads.append(threading.Thread(target=mutator)) + + with threading_helper.start_threads(threads): + pass + + def test_pickle_dumps_with_concurrent_list_mutations(self): + # gh-149816: Pickling a list while another thread mutates it + # used to be a UAF in free-threaded mode. batch_list_exact() + # used PyList_GET_ITEM (borrowed) followed by Py_INCREF, and a + # concurrent replace/pop could free the item between those two + # operations. + shared = [list(range(20)) for _ in range(50)] + + def dumper(): + for _ in range(1000): + try: + pickle.dumps(shared) + except (RuntimeError, IndexError): + pass + + def mutator(): + for i in range(1000): + idx = i % 50 + shared[idx] = list(range(i % 20)) + if i % 10 == 0: + try: + shared.pop() + except IndexError: + pass + shared.append([i]) + + threads = [] + for _ in range(10): + threads.append(threading.Thread(target=dumper)) + threads.append(threading.Thread(target=mutator)) + + with threading_helper.start_threads(threads): + pass + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index a3b9f4b0175ae76..a73525e1bebfb49 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -16,18 +16,19 @@ def run_in_threads(targets): thread.join() +class Spam: + __slots__ = [ + "eggs", + ] + + def __init__(self, initial_value): + self.eggs = initial_value + + @threading_helper.requires_working_threading() class TestSlots(TestCase): def test_object(self): - class Spam: - __slots__ = [ - "eggs", - ] - - def __init__(self, initial_value): - self.eggs = initial_value - spam = Spam(0) iters = 20_000 @@ -43,6 +44,24 @@ def reader(): run_in_threads([writer, reader, reader, reader]) + def test_del_object_is_atomic(self): + # Testing whether the implementation of `del slots_object.attribute` + # removes the attribute atomically, thus avoiding non-sequentially- + # consistent behaviors. + # https://github.com/python/cpython/issues/146270 + def deleter(spam, successes): + try: + del spam.eggs + successes.append(True) + except AttributeError: + successes.append(False) + + for _ in range(10): + spam = Spam(0) + successes = [] + threading_helper.run_concurrently(deleter, nthreads=4, args=(spam, successes)) + self.assertEqual(sum(successes), 1) + def test_T_BOOL(self): spam_old = _testcapi._test_structmembersType_OldAPI() spam_new = _testcapi._test_structmembersType_NewAPI() diff --git a/Lib/test/test_free_threading/test_sys.py b/Lib/test/test_free_threading/test_sys.py new file mode 100644 index 000000000000000..37b53bd723fd767 --- /dev/null +++ b/Lib/test/test_free_threading/test_sys.py @@ -0,0 +1,28 @@ +import sys +import unittest +from test.support import threading_helper + + +class SysModuleTest(unittest.TestCase): + def test_int_max_str_digits_thread(self): + # gh-151218: Check that it's safe to call get_int_max_str_digits() and + # set_int_max_str_digits() in parallel. Previously, this test triggered + # warnings in TSan on a free threaded build. + + old_limit = sys.get_int_max_str_digits() + self.addCleanup(sys.set_int_max_str_digits, old_limit) + + def worker(worker_id): + if not worker_id: + for i in range (20_000): + sys.get_int_max_str_digits() + else: + for i in range (20_000): + sys.set_int_max_str_digits(4300 + (i & 7)) + + workers = [lambda: worker(i) for i in range(5)] + threading_helper.run_concurrently(workers) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 05d0cbd2445c4cb..5cc02c30ec2ba33 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -593,6 +593,17 @@ def test_compile_time_concat_errors(self): r"""b'' f''""", ]) + def test_concat_decode_failure_does_not_crash(self): + script = r''' +import builtins +builtins.__import__ = builtins # Breaks warning machinery so _get_resized_exprs returns NULL +try: + compile('"x"f"\]"b""', '', 'exec') +except Exception: + pass +''' + assert_python_ok('-c', script) + def test_literal(self): self.assertEqual(f'', '') self.assertEqual(f'a', 'a') diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c864d401f9ed67e..983a8b92cf6384d 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -8,6 +8,7 @@ import io import errno import os +import sys import threading import time import unittest @@ -16,7 +17,7 @@ except ImportError: ssl = None -from unittest import TestCase, skipUnless +from unittest import mock, TestCase, skipUnless from test import support from test.support import requires_subprocess from test.support import threading_helper @@ -590,6 +591,8 @@ def test_quit(self): # Ensure the connection gets closed; sock attribute should be None self.assertEqual(self.client.sock, None) + @unittest.skipIf(sys.platform == 'cygwin', + "MSG_OOB doesn't work properly on Cygwin") def test_abort(self): self.client.abort() @@ -1145,6 +1148,40 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + def _make_pair(self, *, advertised_host, real_host, trust=False): + source = mock.Mock(spec=ftplib.FTP) + source.trust_server_pasv_ipv4_address = trust + source.sock.getpeername.return_value = (real_host, 21) + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source.sendcmd.side_effect = lambda cmd: ( + f'227 Entering Passive Mode ({advertised},1,2).' + if cmd == 'PASV' else '150 ok') + target = mock.Mock(spec=ftplib.FTP) + target.sendcmd.return_value = '150 ok' + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('198.51.100.7', 258) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('10.0.0.5', 258) + + class MiscTestCase(TestCase): def test__all__(self): not_exported = { diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index a8ee7d119e4bc61..c30386afe41849b 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1134,6 +1134,14 @@ def add(x, y): self.assertRaises(TypeError, self.reduce, add, [0, 1], initial="") self.assertEqual(self.reduce(42, "", initial="1"), "1") # func is never called with one item + def test_reduce_with_kwargs(self): + with self.assertRaises(TypeError): + self.reduce(function=lambda x, y: (x or 1) + y, sequence=[1, 2, 3, 4, 5]) + with self.assertRaises(TypeError): + self.reduce(function=lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) + with self.assertRaises(TypeError): + self.reduce(lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestReduceC(TestReduce, unittest.TestCase): @@ -1144,12 +1152,6 @@ class TestReduceC(TestReduce, unittest.TestCase): class TestReducePy(TestReduce, unittest.TestCase): reduce = staticmethod(py_functools.reduce) - def test_reduce_with_kwargs(self): - with self.assertWarns(DeprecationWarning): - self.reduce(function=lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) - with self.assertWarns(DeprecationWarning): - self.reduce(lambda x, y: x + y, sequence=[1, 2, 3, 4, 5], initial=1) - class TestCmpToKey: diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 88d265cbc21709d..3fc084ea6e9c6e9 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1288,6 +1288,15 @@ def test_tuple_untrack_counts(self): # Use n // 2 just in case some other objects were collected. self.assertTrue(new_count - count > (n // 2)) + @requires_gil_enabled('need generational GC') + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_heap_size(self): + count = _testinternalcapi.get_tracked_heap_size() + l = [] + self.assertEqual(count + 1, _testinternalcapi.get_tracked_heap_size()) + del l + self.assertEqual(count, _testinternalcapi.get_tracked_heap_size()) + class GCCallbackTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_gc_stats.py b/Lib/test/test_gc_stats.py index 59365ad45b32c9f..bd75924397e76e8 100644 --- a/Lib/test/test_gc_stats.py +++ b/Lib/test/test_gc_stats.py @@ -22,7 +22,7 @@ GC_STATS_FIELDS = ( "gen", "iid", "ts_start", "ts_stop", "collections", "collected", - "uncollectable", "candidates", "duration") + "uncollectable", "candidates", "heap_size", "duration") def get_interpreter_identifiers(gc_stats) -> tuple[int,...]: diff --git a/Lib/test/test_gdb/test_jit.py b/Lib/test/test_gdb/test_jit.py index ea88d7b0a1fe759..7cb604f8aee8c3e 100644 --- a/Lib/test/test_gdb/test_jit.py +++ b/Lib/test/test_gdb/test_jit.py @@ -4,9 +4,18 @@ import sys import unittest +from test.support import import_helper + from .util import setup_module, DebuggerTests +_testinternalcapi = import_helper.import_module("_testinternalcapi") +NATIVE_JIT_ENABLED = ( + hasattr(sys, "_jit") + and sys._jit.is_enabled() + and _testinternalcapi.get_jit_backend() == "jit" +) + JIT_SAMPLE_SCRIPT = os.path.join(os.path.dirname(__file__), "gdb_jit_sample.py") # In batch GDB, break in builtin_id() while it is running under JIT, # then repeatedly "finish" until the selected frame is the JIT executor. @@ -62,14 +71,14 @@ def setUpModule(): # Python/jit_unwind.c, and the synthetic EH-frame is only implemented for # x86_64 and AArch64 (a #error fires otherwise). Skip cleanly on other # platforms or architectures instead of producing timeouts / empty backtraces. -# is_enabled() implies is_available() and also implies that the runtime has -# JIT execution active; interpreter-only tier 2 builds don't hit this path. +# sys._jit.is_enabled() is true for --enable-experimental-jit=interpreter, +# but these tests need native JIT code and a py::jit:executor frame. @unittest.skipUnless(sys.platform == "linux", "GDB JIT interface is only implemented for Linux + ELF") @unittest.skipUnless(platform.machine() in ("x86_64", "aarch64"), "GDB JIT CFI emitter only supports x86_64 and AArch64") -@unittest.skipUnless(hasattr(sys, "_jit") and sys._jit.is_enabled(), - "requires a JIT-enabled build with JIT execution active") +@unittest.skipUnless(NATIVE_JIT_ENABLED, + "requires native JIT execution active") class JitBacktraceTests(DebuggerTests): def get_stack_trace(self, **kwargs): # These tests validate the JIT-relevant part of the backtrace via diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 748309b54593a18..8ce5370e922e069 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -369,8 +369,9 @@ def test_sync_sp(self): stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = Py_None; stack_pointer[0] = res; stack_pointer += 1; @@ -392,8 +393,9 @@ def test_sync_sp(self): res = Py_None; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -626,8 +628,9 @@ def test_macro_instruction(self): uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); op1(left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 2 cache entries */ // OP2 @@ -635,9 +638,10 @@ def test_macro_instruction(self): arg2 = stack_pointer[-3]; uint32_t extra = read_u32(&this_instr[4].cache); (void)extra; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); res = op2(arg2, left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer[-3] = res; stack_pointer += -2; @@ -662,8 +666,9 @@ def test_macro_instruction(self): uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); op1(left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -685,8 +690,9 @@ def test_macro_instruction(self): left = stack_pointer[-2]; arg2 = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res = op3(arg2, left, right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer[-3] = res; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1475,8 +1481,10 @@ def test_stack_save_reload(self): next_instr += 1; INSTRUCTION_STATS(BALANCED); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); code(); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -1500,6 +1508,10 @@ def test_stack_save_reload_paired(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -1522,13 +1534,13 @@ def test_stack_reload_only(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BALANCED); - _PyFrame_SetStackPointer(frame, stack_pointer); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); DISPATCH(); } """ - with self.assertRaises(SyntaxError): - self.run_cases_test(input, output) + self.run_cases_test(input, output) def test_stack_save_only(self): @@ -1614,16 +1626,19 @@ def test_escaping_call_next_to_cmacro(self): INSTRUCTION_STATS(OP); #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #else _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); another_escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #endif - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); yet_another_escaping_call(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -1735,8 +1750,9 @@ def test_complex_label(self): LABEL(my_label) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); do_thing(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (complex) { JUMP_TO_LABEL(other_label); } @@ -1762,12 +1778,14 @@ def test_spilled_label(self): LABEL(one) { stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(two); } LABEL(two) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(one); } """ @@ -1817,16 +1835,18 @@ def test_multiple_labels(self): LABEL(my_label_1) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); do_thing1(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(my_label_2); } LABEL(my_label_2) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); do_thing2(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(my_label_1); } """ @@ -1878,14 +1898,16 @@ def test_reassigning_dead_inputs(self): _PyStackRef in; in = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); temp = use(in); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); in = temp; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(in); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } """ @@ -2058,6 +2080,42 @@ def test_recording_uop_between_real_uops_rejected(self): "non-recording, non-specializing uops"): self.run_cases_test(input, "") + def test_escaping_in_loop(self): + input = """ + inst(TEST_LOOP_SPILL, (a -- a)) { + int n = (int)oparg; + for (int i = 0; i < n; i++) { + escaping_inside_loop(a); + } + escaping_after_loop(a); + } + """ + output = """ + TARGET(TEST_LOOP_SPILL) { + #if _Py_TAIL_CALL_INTERP + int opcode = TEST_LOOP_SPILL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(TEST_LOOP_SPILL); + _PyStackRef a; + a = stack_pointer[-1]; + int n = (int)oparg; + for (int i = 0; i < n; i++) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + escaping_inside_loop(a); + _PyFrame_StackPointerInvalidate(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + escaping_after_loop(a); + _PyFrame_StackPointerInvalidate(frame); + DISPATCH(); + } + """ + self.run_cases_test(input, output) class TestRecorderTableGeneration(unittest.TestCase): @@ -2181,12 +2239,81 @@ def test_family_member_needs_transform_only_when_shape_changes(self): output = self.generate_tables(input) self.assert_slot_map_lines( output, - "[OP_RAW] = {1, 1, {0}}", - "[OP_RAW_SPECIALIZED] = {1, 0, {0}}", + "[OP_RAW] = {1, 0, {0}}", + "[OP_RAW_SPECIALIZED] = {1, 1, {0}}", "[OP_TYPED] = {1, 0, {0}}", "[OP_TYPED_SPECIALIZED] = {1, 0, {0}}", ) + def test_record_transform_generated_from_recording_uop(self): + input = """ + tier2 op(_RECORD_TOS, (tos -- tos)) { + RECORD_VALUE(PyStackRef_AsPyObjectBorrow(tos)); + } + tier2 op(_RECORD_TOS_TYPE, (tos -- tos)) { + RECORD_VALUE(Py_TYPE(PyStackRef_AsPyObjectBorrow(tos))); + } + op(_DO_STUFF, (tos -- res)) { + res = tos; + } + macro(OP) = _RECORD_TOS + _DO_STUFF; + macro(OP_SPECIALIZED) = _RECORD_TOS_TYPE + _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + output = self.generate_tables(input) + self.assertIn("_PyOpcode_RecordTransform_TOS_TYPE", output) + self.assertIn("tos = PyStackRef_FromPyObjectBorrow(recorded_value);", output) + self.assertIn( + "transformed_value = (PyObject *)Py_TYPE(PyStackRef_AsPyObjectBorrow(tos));", + output, + ) + self.assertIn("return _PyOpcode_RecordTransform_TOS_TYPE(value);", output) + self.assertNotIn("record_trace_transform_to_type", output) + + def test_record_transform_generated_when_only_specialization_records(self): + input = """ + tier2 op(_RECORD_TOS_TYPE, (tos -- tos)) { + RECORD_VALUE(Py_TYPE(PyStackRef_AsPyObjectBorrow(tos))); + } + op(_DO_STUFF, (tos -- res)) { + res = tos; + } + macro(OP) = _DO_STUFF; + macro(OP_SPECIALIZED) = _RECORD_TOS_TYPE + _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + output = self.generate_tables(input) + # Family head must adopt the specialization's recorder. + self.assertIn("[OP] = {1, {_RECORD_TOS_TYPE_INDEX}}", output) + self.assertIn("[OP_SPECIALIZED] = {1, {_RECORD_TOS_TYPE_INDEX}}", output) + # Specialization consumes the slot directly (mask 0), no transform. + self.assert_slot_map_lines(output, "[OP_SPECIALIZED] = {1, 0, {0}}") + self.assertNotIn("_PyOpcode_RecordTransform_TOS_TYPE(", output) + + def test_no_record_transform_when_only_base_records(self): + input = """ + tier2 op(_RECORD_TOS, (tos -- tos)) { + RECORD_VALUE(PyStackRef_AsPyObjectBorrow(tos)); + } + op(_DO_STUFF, (tos -- res)) { + res = tos; + } + macro(OP) = _RECORD_TOS + _DO_STUFF; + macro(OP_SPECIALIZED) = _DO_STUFF; + family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; + """ + output = self.generate_tables(input) + # Family head records via _RECORD_TOS. + self.assertIn("[OP] = {1, {_RECORD_TOS_INDEX}}", output) + self.assertIn("[OP_SPECIALIZED] = {1, {_RECORD_TOS_INDEX}}", output) + # Specialization has no consumer slot map entry (it doesn't read it). + self.assertNotIn( + "[OP_SPECIALIZED] = {1,", self.get_slot_map_section(output) + ) + # No transform helpers are generated. + self.assertNotIn("_PyOpcode_RecordTransform_TOS(", output) + self.assertNotIn("_PyOpcode_RecordTransform_TOS_TYPE", output) + def test_family_member_maps_positional_recorders_to_family_slots(self): input = """ tier2 op(_RECORD_TOS, (sub -- sub)) { @@ -2227,8 +2354,8 @@ def test_family_member_maps_non_positional_recorders_by_stack_shape(self): output = self.generate_tables(input) self.assert_slot_map_lines( output, - "[OP] = {1, 1, {0}}", - "[OP_SPECIALIZED] = {1, 0, {0}}", + "[OP] = {1, 0, {0}}", + "[OP_SPECIALIZED] = {1, 1, {0}}", ) def test_family_head_records_union_of_member_recorders(self): @@ -2243,7 +2370,12 @@ def test_family_head_records_union_of_member_recorders(self): macro(OP_SPECIALIZED) = _RECORD_TOS + _DO_STUFF; family(OP, INLINE_CACHE_ENTRIES_OP) = { OP_SPECIALIZED }; """ + analysis = self.analyze_input(input) output = self.generate_tables(input) + self.assertEqual( + analysis.families["OP"].get_member_record_names(), + ("_RECORD_TOS",), + ) self.assertIn("[OP] = {1, {_RECORD_TOS_INDEX}}", output) self.assertIn("[OP_SPECIALIZED] = {1, {_RECORD_TOS_INDEX}}", output) self.assert_slot_map_lines(output, "[OP_SPECIALIZED] = {1, 0, {0}}") @@ -2277,12 +2409,12 @@ def test_family_detects_base_and_specialized_recording_difference(self): ), ["_RECORD_TOS_TYPE"], ) - self.assertIn("[OP] = {1, {_RECORD_TOS_TYPE_INDEX}}", output) - self.assertIn("[OP_SPECIALIZED] = {1, {_RECORD_TOS_TYPE_INDEX}}", output) + self.assertIn("[OP] = {1, {_RECORD_TOS_INDEX}}", output) + self.assertIn("[OP_SPECIALIZED] = {1, {_RECORD_TOS_INDEX}}", output) self.assert_slot_map_lines( output, - "[OP] = {1, 1, {0}}", - "[OP_SPECIALIZED] = {1, 0, {0}}", + "[OP] = {1, 0, {0}}", + "[OP_SPECIALIZED] = {1, 1, {0}}", ) def test_family_head_falls_back_for_missing_member_slots(self): diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index a5969b7a47d948b..7816775620bc013 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -55,15 +55,14 @@ from unittest.case import _AssertRaisesContext from queue import Queue, SimpleQueue from weakref import WeakSet, ReferenceType, ref -import typing -from typing import Unpack try: from tkinter import Event except ImportError: Event = None from string.templatelib import Template, Interpolation -from typing import TypeVar +import typing +from typing import TypeVar, Unpack T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') @@ -621,6 +620,14 @@ def test_nested_paramspec_specialization(self): self.assertEqual(deeply_nested_specialized.__args__, ([str, [float], int], float)) self.assertEqual(deeply_nested_specialized.__parameters__, ()) + def test_gh150146(self): + # It used to crash: + for container in [memoryview, list, tuple]: + with self.subTest(container=container): + x = container[TypeVar("")] + with self.assertRaises(TypeError): + x[*typing.Mapping[..., ...]] + class TypeIterationTests(unittest.TestCase): _UNITERABLE_TYPES = (list, tuple) diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index e52e17b8dc73667..f08b9328a9ff042 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -1,5 +1,8 @@ """Test script for the grp module.""" +import random +import string +import sys import unittest from test.support import import_helper @@ -33,7 +36,15 @@ def test_values_extended(self): self.skipTest('huge group file, extended test skipped') for e in entries: - e2 = grp.getgrgid(e.gr_gid) + try: + e2 = grp.getgrgid(e.gr_gid) + except KeyError: + # On Cygwin, some groups returned by getgrall() cannot be + # retrieved by getgrgid() + if sys.platform == 'cygwin': + continue + raise + self.check_value(e2) self.assertEqual(e2.gr_gid, e.gr_gid) name = e.gr_name @@ -50,61 +61,51 @@ def test_values_extended(self): def test_errors(self): self.assertRaises(TypeError, grp.getgrgid) self.assertRaises(TypeError, grp.getgrgid, 3.14) + self.assertRaises(TypeError, grp.getgrgid, 0.0) + self.assertRaises(TypeError, grp.getgrgid, 0, 0) + # should be out of gid_t range + self.assertRaises(OverflowError, grp.getgrgid, 2**128) + self.assertRaises(OverflowError, grp.getgrgid, -2**128) self.assertRaises(TypeError, grp.getgrnam) self.assertRaises(TypeError, grp.getgrnam, 42) - self.assertRaises(TypeError, grp.getgrall, 42) + self.assertRaises(TypeError, grp.getgrnam, b'root') + self.assertRaises(TypeError, grp.getgrnam, 'root', 0) # embedded null character self.assertRaisesRegex(ValueError, 'null', grp.getgrnam, 'a\x00b') + self.assertRaisesRegex(ValueError, 'null', grp.getgrnam, 'root\x00') + self.assertRaises(UnicodeEncodeError, grp.getgrnam, 'roo\udc74') + self.assertRaises(KeyError, grp.getgrnam, '') + self.assertRaises(TypeError, grp.getgrall, 42) - # try to get some errors - bynames = {} - bygids = {} - for (n, p, g, mem) in grp.getgrall(): - if not n or n == '+': - continue # skip NIS entries etc. - bynames[n] = g - bygids[g] = n - - allnames = list(bynames.keys()) - namei = 0 - fakename = allnames[namei] - while fakename in bynames: - chars = list(fakename) - for i in range(len(chars)): - if chars[i] == 'z': - chars[i] = 'A' - break - elif chars[i] == 'Z': - continue + # Find a non-existent group name. + # getgrall() will not necessarily report all existing groups + # (typical for LDAP based directories in big organizations). + for _ in range(30): + fakename = ''.join(random.choices(string.ascii_lowercase, k=6)) + try: + grp.getgrnam(fakename) + except KeyError: + break + else: + self.fail('Cannot find non-existent group name') + + # Find a non-existent gid. + maxgid = 2**31 + for _ in range(30): + fakegid = random.randrange(maxgid) + try: + grp.getgrgid(fakegid) + except KeyError: + break + except OverflowError: + if maxgid == 2**31: + maxgid = 2**16-1 + elif maxgid == 2**16-1: + maxgid = 2**15 else: - chars[i] = chr(ord(chars[i]) + 1) - break - else: - namei = namei + 1 - try: - fakename = allnames[namei] - except IndexError: - # should never happen... if so, just forget it - break - fakename = ''.join(chars) - - self.assertRaises(KeyError, grp.getgrnam, fakename) - - # Choose a non-existent gid. - fakegid = 4127 - while fakegid in bygids: - fakegid = (fakegid * 3) % 0x10000 - - self.assertRaises(KeyError, grp.getgrgid, fakegid) - - def test_noninteger_gid(self): - entries = grp.getgrall() - if not entries: - self.skipTest('no groups') - # Choose an existent gid. - gid = entries[0][2] - self.assertRaises(TypeError, grp.getgrgid, float(gid)) - self.assertRaises(TypeError, grp.getgrgid, str(gid)) + raise + else: + self.fail('Cannot find non-existent gid') if __name__ == "__main__": diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 442d30fc970fa94..8bc8e507683cb50 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -10,6 +10,7 @@ import sys import unittest from subprocess import PIPE, Popen +from unittest import mock from test.support import catch_unraisable_exception from test.support import force_not_colorized_test_class, import_helper from test.support import os_helper @@ -350,6 +351,37 @@ def test_mtime(self): self.assertEqual(dataRead, data1) self.assertEqual(fRead.mtime, mtime) + def test_mtime_with_open(self): + mtime = 123456789 + with gzip.open(self.filename, "wb", mtime=mtime) as fWrite: + fWrite.write(data1) + with gzip.open(self.filename, "rb") as fRead: + self.assertTrue(hasattr(fRead, 'mtime')) + self.assertIsNone(fRead.mtime) + dataRead = fRead.read() + self.assertEqual(dataRead, data1) + self.assertEqual(fRead.mtime, mtime) + + def test_mtime_out_of_range(self): + for mtime in (-1, 2**32): + with gzip.GzipFile(self.filename, 'w', mtime=mtime) as fWrite: + fWrite.write(data1) + with gzip.GzipFile(self.filename) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + datac = gzip.compress(data1, mtime=mtime) + with gzip.GzipFile(fileobj=io.BytesIO(datac)) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + + for mtime in (-1, 2**32): + with mock.patch('time.time', return_value=float(mtime)): + with gzip.GzipFile(self.filename, 'w') as fWrite: + fWrite.write(data1) + with gzip.GzipFile(self.filename) as fRead: + fRead.read(1) + self.assertEqual(fRead.mtime, 0) + def test_metadata(self): mtime = 123456789 @@ -795,6 +827,35 @@ def test_decompress_missing_trailer(self): compressed_data = gzip.compress(data1) self.assertRaises(EOFError, gzip.decompress, compressed_data[:-8]) + def test_truncated_header(self): + truncated_headers = [ + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00", # Missing OS byte + b"\x1f\x8b\x08\x02\x00\x00\x00\x00\x00\xff", # FHRC, but no checksum + b"\x1f\x8b\x08\x04\x00\x00\x00\x00\x00\xff", # FEXTRA, but no xlen + b"\x1f\x8b\x08\x04\x00\x00\x00\x00\x00\xff\xaa\x00", # FEXTRA, xlen, but no data + b"\x1f\x8b\x08\x08\x00\x00\x00\x00\x00\xff", # FNAME but no fname + b"\x1f\x8b\x08\x10\x00\x00\x00\x00\x00\xff", # FCOMMENT, but no fcomment + ] + for header in truncated_headers: + with self.subTest(header=header): + with self.assertRaises(EOFError): + gzip.decompress(header) + + def test_corrupted_gzip_header(self): + header = (b"\x1f\x8b\x08\x1f\x00\x00\x00\x00\x00\xff" # All flags set + b"\x05\x00" # Xlen = 5 + b"extra" + b"name\x00" + b"comment\x00") + true_crc = zlib.crc32(header) & 0xFFFF + corrupted_crc = true_crc ^ 0xFFFF + corrupted_header = header + corrupted_crc.to_bytes(2, "little") + with self.assertRaises(gzip.BadGzipFile) as err: + gzip.decompress(corrupted_header) + self.assertEqual(str(err.exception), + f"Corrupted gzip header. Checksums do not " + f"match: {true_crc:04x} != {corrupted_crc:04x}") + def test_read_truncated(self): data = data1*50 # Drop the CRC (4 bytes) and file size (4 bytes). diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index cde268e32418509..d1df2ec42f0d146 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,11 +1,11 @@ # Simple test suite for http/cookies.py -import base64 import copy import unittest import doctest from http import cookies import pickle from test import support +import urllib.parse class CookieTests(unittest.TestCase): @@ -152,21 +152,21 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme; Version=1').decode('ascii') + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme; Version=1', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(), fr""" """) - cookie_encoded = base64.b64encode(b'Customer="WILE_E_COYOTE"; Path=/acme').decode('ascii') + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(['path']), fr""" """) @@ -271,21 +271,21 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1').decode('ascii') + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(), fr""" """) - expected_encoded_cookie = base64.b64encode(b'Customer=\"WILE_E_COYOTE\"; Path=/acme').decode('ascii') + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme', safe='', encoding='utf-8') with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): self.assertEqual(C.js_output(['path']), fr""" """) @@ -376,13 +376,14 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) - expected_encoded_cookie = base64.b64encode( - ("%s=%s; Path=/foo" % (i, "%s_coded_val" % i)).encode("ascii") - ).decode('ascii') + expected_encoded_cookie = urllib.parse.quote( + "%s=%s; Path=/foo" % (i, "%s_coded_val" % i), + safe='', encoding='utf-8', + ) expected_js_output = """ """ % (expected_encoded_cookie,) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 574529c70741d8f..d4ae032610a91e2 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -824,7 +824,7 @@ def test_get(self): # chmod() doesn't work as expected on Windows, and filesystem # permissions are ignored by root on Unix. - if os.name == 'posix' and os.geteuid() != 0: + if os.name == 'posix' and os.geteuid() != 0 and sys.platform != 'cygwin': os.chmod(self.tempdir, 0) try: response = self.request(self.base_url + '/') diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index cb5454b40eccf90..fb256fb7cbcd344 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -434,6 +434,16 @@ def cmd_AUTHENTICATE(self, tag, args): r'\[AUTHENTICATIONFAILED\] invalid'): client.authenticate('MYAUTH', lambda x: b'fake') + def test_invalid_login(self): + class MyServer(SimpleIMAPHandler): + def cmd_LOGIN(self, tag, args): + self.server.logged = args[0] + self._send_tagged(tag, 'NO', '[LOGIN] failed') + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'\[LOGIN\] failed'): + client.login('user', 'wrongpass') + def test_valid_authentication_bytes(self): class MyServer(SimpleIMAPHandler): def cmd_AUTHENTICATE(self, tag, args): @@ -665,11 +675,33 @@ def test_control_characters(self): # property tests - def test_file_property_should_not_be_accessed(self): + def test_file_property_getter(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(client.file.raw, socket.SocketIO) + + def test_file_property_setter(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertWarns(DeprecationWarning): + # ensure that the caller closes the existing file + client.file.close() + for new_file in [mock.Mock(), None]: + with self.assertWarns(DeprecationWarning): + client.file = new_file + with self.assertWarns(DeprecationWarning): + self.assertIs(client.file, new_file) + + def test_file_property_setter_should_not_close_previous_file(self): client, _ = self._setup(SimpleIMAPHandler) - # the 'file' property replaced a private attribute that is now unsafe - with self.assertWarns(RuntimeWarning): - client.file + with mock.patch.object(client, "_imaplib_file", mock.Mock()) as f: + f.close.assert_not_called() + with self.assertWarns(DeprecationWarning): + self.assertIs(client.file, f) + with self.assertWarns(DeprecationWarning): + client.file = None + with self.assertWarns(DeprecationWarning): + self.assertIsNone(client.file) + f.close.assert_not_called() class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index c905c0da0a12327..9f3df8010d32339 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -364,6 +364,15 @@ def test_import_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): import something_that_should_not_exist_anywhere + def test_import_null_byte_in_name_raises_ModuleNotFoundError(self): + # gh-150633: module names containing null bytes should not + # lead to duplicates in sys.modules + before = set(sys.modules) + with self.assertRaises(ModuleNotFoundError): + __import__('zipimport\x00junk') + + self.assertEqual(set(sys.modules), before) + def test_from_import_missing_module_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): from something_that_should_not_exist_anywhere import blah diff --git a/Lib/test/test_importlib/resources/util.py b/Lib/test/test_importlib/resources/util.py index 85b5c61518de444..e734746e9dcb3b8 100644 --- a/Lib/test/test_importlib/resources/util.py +++ b/Lib/test/test_importlib/resources/util.py @@ -6,6 +6,7 @@ import pathlib import sys import types +from importlib.abc import Loader, MetaPathFinder from importlib.machinery import ModuleSpec from importlib.resources.abc import ResourceReader, Traversable, TraversableResources @@ -218,7 +219,7 @@ def augment_sys_metapath(self, module): yield sys.meta_path.remove(finder_instance) - class MemoryFinder(importlib.abc.MetaPathFinder): + class MemoryFinder(MetaPathFinder): def __init__(self, module): self._module = module @@ -232,7 +233,7 @@ def find_spec(self, fullname, path, target=None): is_package=True, ) - class MemoryLoader(importlib.abc.Loader): + class MemoryLoader(Loader): def __init__(self, module): self._module = module diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index efe9d27e3407ff1..7351f97fd9a4b5c 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -1355,9 +1355,10 @@ def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, varkw_e=None, defaults_e=None, posonlyargs_e=[], kwonlyargs_e=[], kwonlydefaults_e=None, - ann_e={}): + ann_e={}, + annotation_format=Format.VALUE): args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - inspect.getfullargspec(routine) + inspect.getfullargspec(routine, annotation_format=annotation_format) self.assertEqual(args, args_e) self.assertEqual(varargs, varargs_e) self.assertEqual(varkw, varkw_e) @@ -1390,6 +1391,19 @@ def test_getfullargspec(self): kwonlyargs_e=['e', 'f'], kwonlydefaults_e={'e': 4, 'f': 5}) + def get_getfullargspec_with_undefined_names_in_annotations(self): + def my_func(a: undefined_name): + pass + + with self.assertRaises(NameError): + inspect.getfullargspec(my_func) + + self.assertFullArgSpecEquals(my_func, ['a'], ann_e={'a': 'undefined_name'}, + annotation_format=Format.STRING) + + arg_spec = inspect.getfullargspec(my_func, annotation_format=Format.FORWARDREF) + self.assertIsInstance(arg_spec.annotations['a'], ForwardRef) + def test_argspec_api_ignores_wrapped(self): # Issue 20684: low level introspection API must ignore __wrapped__ @functools.wraps(mod.spam) @@ -6241,8 +6255,7 @@ def test_faulthandler_module_has_signatures(self): self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) def test_functools_module_has_signatures(self): - unsupported_signature = {"reduce"} - self._test_module_has_signatures(functools, unsupported_signature=unsupported_signature) + self._test_module_has_signatures(functools) def test_gc_module_has_signatures(self): import gc diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index 482b183da23ffa2..3669ac0b038b71b 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -967,6 +967,25 @@ def test_setstate(self): memio.close() self.assertRaises(ValueError, memio.__setstate__, ("closed", "", 0, None)) + def test_write_str_subclass(self): + # Writing a str subclass should use the subclass's unicode data + # directly, not call __str__ on it (which may return a different + # value). gh-149047 + class MyStr(str): + def __str__(self): + return "WRONG" + + s = MyStr("correct") + memio = self.ioclass() + memio.write(s) + self.assertEqual(memio.getvalue(), "correct") + + # Also test the fast path where pos == string_size (STATE_ACCUMULATING) + memio2 = self.ioclass() + memio2.write(MyStr("hello ")) + memio2.write(MyStr("world")) + self.assertEqual(memio2.getvalue(), "hello world") + class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation diff --git a/Lib/test/test_io/test_textio.py b/Lib/test/test_io/test_textio.py index d725f9212ceaaef..82096ab09873955 100644 --- a/Lib/test/test_io/test_textio.py +++ b/Lib/test/test_io/test_textio.py @@ -1560,6 +1560,56 @@ def closed(self): wrapper = self.TextIOWrapper(raw) wrapper.close() # should not crash + def test_reentrant_detach_during_flush(self): + # gh-143008: Reentrant detach() during flush should not crash. + + class DetachOnce(self.BufferedRandom): + wrapper = None + + def detach_once(self): + original = self.wrapper + self.wrapper = None + if original is not None: + original.detach() + original.flush() + + class DetachOnFlush(DetachOnce): + def flush(self): + self.detach_once() + + class DetachOnWrite(DetachOnce): + def write(self, b): + self.detach_once() + return len(b) + + # Separate reference for after detach_once. + wrapper = None + + def make_text(buffer): + nonlocal wrapper + buffer.wrapper = self.TextIOWrapper(buffer, encoding='utf-8') + wrapper = buffer.wrapper + + # Many calls could result in the same null self->buffer crash. + tests = [ + ('truncate', lambda: wrapper.truncate(0)), + ('close', lambda: wrapper.close()), + ('detach', lambda: wrapper.detach()), + ('seek', lambda: wrapper.seek(0)), + ('tell', lambda: wrapper.tell()), + ('reconfigure', lambda: wrapper.reconfigure(line_buffering=True)), + ] + for name, method in tests: + with self.subTest(name): + make_text(DetachOnFlush(self.MockRawIO())) + self.assertRaisesRegex(ValueError, "detached", method) + + # Should not crash. + with self.subTest('read via writeflush'): + make_text(DetachOnWrite(self.MockRawIO())) + wrapper.write('x') + self.assertRaisesRegex(ValueError, "detached", wrapper.read) + class PyTextIOWrapperTest(TextIOWrapperTest, PyTestCase): shutdown_error = "LookupError: unknown encoding: ascii" diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py index 277d2fc99eaec65..0bcabd579a1b309 100644 --- a/Lib/test/test_ioctl.py +++ b/Lib/test/test_ioctl.py @@ -139,6 +139,7 @@ def setUp(self): self.addCleanup(os.close, self.master_fd) @unittest.skipUnless(hasattr(termios, 'TCFLSH'), 'requires termios.TCFLSH') + @unittest.skipIf(sys.platform == 'cygwin', 'test failed on Cygwin') def test_ioctl_clear_input_or_output(self): wfd = self.slave_fd rfd = self.master_fd diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index d846c8af7ec4345..1d51fb2de0e69e4 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -87,6 +87,13 @@ def test_array_hook(self): self.assertEqual(self.loads('[]', array_hook=tuple), ()) + def test_load_array_hook(self): + # json.load must forward array_hook to loads + fp = StringIO('[10, 20, 30]') + result = self.json.load(fp, array_hook=tuple) + self.assertEqual(result, (10, 20, 30)) + self.assertEqual(type(result), tuple) + def test_decoder_optimizations(self): # Several optimizations were made that skip over calls to # the whitespace regex, so this test is designed to try and diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index ffd3404e6f77a07..d732fc80cf1cf30 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -68,6 +68,7 @@ def default(self, o): self.fail("didn't raise ValueError on default recursion") + @support.skip_if_pgo_task # fails during PGO training w/ some stack sizes @support.skip_if_unlimited_stack_size @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index c522bc1c2c093c7..795890bd7e4b473 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -469,6 +469,7 @@ def test_search_major_2(self): except subprocess.CalledProcessError: if not is_installed("2.7"): raise unittest.SkipTest("requires at least one Python 2.x install") + raise self.assertEqual("PythonCore", data["env.company"]) self.assertStartsWith(data["env.tag"], "2.") diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index a9a8cd143e0d756..4658882243d65ff 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -10,6 +10,7 @@ import unittest import tempfile import os +import contextlib from test import support from test.support.script_helper import assert_python_ok @@ -20,8 +21,10 @@ _testcapi = None -class LazyImportTests(unittest.TestCase): - """Tests for basic lazy import functionality.""" +class LazyImportTestCase(unittest.TestCase): + def setUp(self): + self.lazy_imports_filter = sys.get_lazy_imports_filter() + self.lazy_imports = sys.get_lazy_imports() def tearDown(self): """Clean up any test modules from sys.modules.""" @@ -29,16 +32,19 @@ def tearDown(self): if key.startswith('test.test_lazy_import.data'): del sys.modules[key] - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") + sys.set_lazy_imports_filter(self.lazy_imports_filter) + sys.set_lazy_imports(self.lazy_imports) sys.lazy_modules.clear() + +class LazyImportTests(LazyImportTestCase): + """Tests for basic lazy import functionality.""" + def test_basic_unused(self): """Lazy imported module should not be loaded if never accessed.""" import test.test_lazy_import.data.basic_unused self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) - self.assertIn("test.test_lazy_import.data", sys.lazy_modules) - self.assertEqual(sys.lazy_modules["test.test_lazy_import.data"], {"basic2"}) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) def test_sys_lazy_modules(self): try: @@ -48,7 +54,7 @@ def test_sys_lazy_modules(self): self.assertFalse("test.test_lazy_import.data.basic2" in sys.modules) self.assertIn("test.test_lazy_import.data", sys.lazy_modules) - self.assertEqual(sys.lazy_modules["test.test_lazy_import.data"], {"basic2"}) + self.assertIn("test.test_lazy_import.data.basic2", sys.lazy_modules) test.test_lazy_import.data.basic_from_unused.basic2 self.assertNotIn("test.test_import.data", sys.lazy_modules) @@ -88,22 +94,87 @@ def test_basic_used(self): import test.test_lazy_import.data.basic_used self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + @support.requires_subprocess() + def test_from_import_with_module_getattr(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) -class GlobalLazyImportModeTests(unittest.TestCase): - """Tests for sys.set_lazy_imports() global mode control.""" + @support.requires_subprocess() + def test_from_import_with_module_getattr_raising(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + lazy from test.test_lazy_import.data.module_with_getattr import raising_attr - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] + try: + raising_attr + except ValueError as exc: + assert str(exc) == 'from_getattr', exc + else: + assert False, f'ValueError is not raised: {raising_attr}' + """) + assert_python_ok("-c", code) - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") + @support.requires_subprocess() + def test_from_import_with_module_getattr_missing(self): + """Lazy from import should respect module-level __getattr__.""" + for attr in ("missing_attr", "import_error_attr"): + with self.subTest(attr=attr): + code = textwrap.dedent(f""" + lazy from test.test_lazy_import.data.module_with_getattr import {attr} + + try: + {attr} + except ImportError as exc: + assert '{attr}' in str(exc), exc + assert exc.__cause__ is not None + else: + assert False, ('ImportError is not raised', {attr}) + """) + assert_python_ok("-c", code) - def test_global_off(self): - """Mode 'none' should disable lazy imports entirely.""" - import test.test_lazy_import.data.global_off - self.assertIn("test.test_lazy_import.data.basic2", sys.modules) + @support.requires_subprocess() + def test_from_import_with_module_getattr_warning(self): + """Lazy from import should respect module-level __getattr__.""" + code = textwrap.dedent(""" + import warnings + + with warnings.catch_warnings(record=True) as log: + lazy from test.test_lazy_import.data.module_with_getattr import warning_attr + + assert log == [] + + with warnings.catch_warnings(record=True) as log: + warning_attr + assert warning_attr == 'from_warning_attr', warning_attr + assert len(log) == 1, log + assert isinstance(log[0].message, UserWarning), log + assert str(log[0].message) == 'from_getattr', log + """) + assert_python_ok("-c", code) + + @support.requires_subprocess() + def test_from_import_with_imported_module_getattr(self): + """Lazy from import should not shadow an imported module's __getattr__.""" + code = textwrap.dedent(""" + import test.test_lazy_import.data.module_with_getattr as mod + lazy from test.test_lazy_import.data.module_with_getattr import dynamic_attr + assert dynamic_attr == "from_getattr" + assert mod.dynamic_attr == "from_getattr" + """) + assert_python_ok("-c", code) + + +class GlobalLazyImportModeTests(LazyImportTestCase): + """Tests for sys.set_lazy_imports() global mode control.""" + + def test_global_off_rejected(self): + """Mode 'none' is not supported.""" + with self.assertRaises(ValueError): + sys.set_lazy_imports("none") def test_global_on(self): """Mode 'all' should make regular imports lazy.""" @@ -131,17 +202,9 @@ def test_global_filter_from_true(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) -class CompatibilityModeTests(unittest.TestCase): +class CompatibilityModeTests(LazyImportTestCase): """Tests for __lazy_modules__ compatibility mode.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_compatibility_mode(self): """__lazy_modules__ should enable lazy imports for listed modules.""" import test.test_lazy_import.data.basic_compatibility_mode @@ -168,17 +231,9 @@ def test_compatibility_mode_relative(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) -class ModuleIntrospectionTests(unittest.TestCase): +class ModuleIntrospectionTests(LazyImportTestCase): """Tests for module dict and getattr behavior with lazy imports.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_modules_dict(self): """Accessing module.__dict__ should not trigger reification.""" import test.test_lazy_import.data.modules_dict @@ -195,17 +250,9 @@ def test_modules_getattr_other(self): self.assertNotIn("test.test_lazy_import.data.basic2", sys.modules) -class LazyImportTypeTests(unittest.TestCase): +class LazyImportTypeTests(LazyImportTestCase): """Tests for the LazyImportType and its resolve() method.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_lazy_value_resolve(self): """resolve() method should force the lazy import to load.""" import test.test_lazy_import.data.lazy_get_value @@ -231,17 +278,9 @@ def test_lazy_import_type_attributes_accessible(self): self.assertIn(b" initial_count print("OK") """) @@ -510,35 +650,22 @@ def test_lazy_modules_tracks_lazy_imports(self): @support.requires_subprocess() -class ErrorHandlingTests(unittest.TestCase): +class ErrorHandlingTests(LazyImportTestCase): """Tests for error handling during lazy import reification. PEP 810: Errors during reification should show exception chaining with both the lazy import definition location and the access location. """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_import_error_shows_chained_traceback(self): - """ImportError during reification should chain to show both definition and access.""" - # Errors at reification must show where the lazy import was defined - # AND where the access happened, per PEP 810 "Reification" section + """Accessing a nonexistent lazy submodule via parent attr raises AttributeError.""" code = textwrap.dedent(""" import sys lazy import test.test_lazy_import.data.nonexistent_module try: x = test.test_lazy_import.data.nonexistent_module - except ImportError as e: - # Should have __cause__ showing the original error - # The exception chain shows both where import was defined and where access happened - assert e.__cause__ is not None, "Expected chained exception" + except AttributeError as e: print("OK") """) result = subprocess.run( @@ -586,7 +713,7 @@ def test_reification_retries_on_failure(self): # First access - should fail try: x = test.test_lazy_import.data.broken_module - except ValueError: + except AttributeError: pass # The lazy object should still be a lazy proxy (not reified) @@ -596,7 +723,7 @@ def test_reification_retries_on_failure(self): # Second access - should also fail (retry the import) try: x = test.test_lazy_import.data.broken_module - except ValueError: + except AttributeError: print("OK - retry worked") """) result = subprocess.run( @@ -609,7 +736,6 @@ def test_reification_retries_on_failure(self): def test_error_during_module_execution_propagates(self): """Errors in module code during reification should propagate correctly.""" - # Module that raises during import should propagate with chaining code = textwrap.dedent(""" import sys lazy import test.test_lazy_import.data.broken_module @@ -617,12 +743,8 @@ def test_error_during_module_execution_propagates(self): try: _ = test.test_lazy_import.data.broken_module print("FAIL - should have raised") - except ValueError as e: - # The ValueError from the module should be the cause - if "always fails" in str(e) or (e.__cause__ and "always fails" in str(e.__cause__)): - print("OK") - else: - print(f"FAIL - wrong error: {e}") + except AttributeError: + print("OK") """) result = subprocess.run( [sys.executable, "-c", code], @@ -667,7 +789,7 @@ def hello(): @support.requires_subprocess() -class GlobalsAndDictTests(unittest.TestCase): +class GlobalsAndDictTests(LazyImportTestCase): """Tests for globals() and __dict__ behavior with lazy imports. PEP 810: "Calling globals() or accessing a module's __dict__ does not trigger @@ -675,14 +797,6 @@ class GlobalsAndDictTests(unittest.TestCase): through that dictionary still returns lazy proxy objects." """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_globals_returns_lazy_proxy_when_accessed_from_function(self): """globals() accessed from a function should return lazy proxy without reification. @@ -837,7 +951,7 @@ def f(): @support.requires_subprocess() -class MultipleNameFromImportTests(unittest.TestCase): +class MultipleNameFromImportTests(LazyImportTestCase): """Tests for lazy from ... import with multiple names. PEP 810: "When using lazy from ... import, each imported name is bound to a @@ -846,14 +960,6 @@ class MultipleNameFromImportTests(unittest.TestCase): Other names remain as lazy proxies until they are accessed." """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_accessing_one_name_leaves_others_as_proxies(self): """Accessing one name from multi-name import should leave others lazy.""" code = textwrap.dedent(""" @@ -918,20 +1024,12 @@ def test_all_names_reified_after_all_accessed(self): @support.requires_subprocess() -class SysLazyModulesTrackingTests(unittest.TestCase): +class SysLazyModulesTrackingTests(LazyImportTestCase): """Tests for sys.lazy_modules tracking behavior. PEP 810: "When the module is reified, it's removed from sys.lazy_modules" """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_module_added_to_lazy_modules_on_lazy_import(self): """Module should be added to sys.lazy_modules when lazily imported.""" # PEP 810 states lazy_modules tracks modules that have been lazily imported @@ -945,15 +1043,14 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): lazy import test.test_lazy_import.data.basic2 # Should be in lazy_modules after lazy import - assert "test.test_lazy_import.data" in sys.lazy_modules - assert sys.lazy_modules["test.test_lazy_import.data"] == {"basic2"} + assert "test.test_lazy_import.data.basic2" in sys.lazy_modules assert len(sys.lazy_modules) > initial_count # Trigger reification _ = test.test_lazy_import.data.basic2.x # Module should still be tracked (for diagnostics per PEP 810) - assert "test.test_lazy_import.data" not in sys.lazy_modules + assert "test.test_lazy_import.data.basic2" not in sys.lazy_modules print("OK") """) result = subprocess.run( @@ -966,8 +1063,8 @@ def test_module_added_to_lazy_modules_on_lazy_import(self): def test_lazy_modules_is_per_interpreter(self): """Each interpreter should have independent sys.lazy_modules.""" - # Basic test that sys.lazy_modules exists and is a dict - self.assertIsInstance(sys.lazy_modules, dict) + # Basic test that sys.lazy_modules exists and is a set + self.assertIsInstance(sys.lazy_modules, set) def test_lazy_module_without_children_is_tracked(self): code = textwrap.dedent(""" @@ -976,10 +1073,6 @@ def test_lazy_module_without_children_is_tracked(self): assert "json" in sys.lazy_modules, ( f"expected 'json' in sys.lazy_modules, got {set(sys.lazy_modules)}" ) - assert sys.lazy_modules["json"] == set(), ( - f"expected empty set for sys.lazy_modules['json'], " - f"got {sys.lazy_modules['json']!r}" - ) print("OK") """) assert_python_ok("-c", code) @@ -1015,24 +1108,16 @@ def test_cli_lazy_imports_all_makes_regular_imports_lazy(self): self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") self.assertIn("LAZY", result.stdout) - def test_cli_lazy_imports_none_forces_all_imports_eager(self): - """-X lazy_imports=none should force all imports to be eager.""" - code = textwrap.dedent(""" - import sys - # Even explicit lazy imports should be eager in 'none' mode - lazy import json - if 'json' in sys.modules: - print("EAGER") - else: - print("LAZY") - """) + def test_cli_lazy_imports_none_is_rejected(self): + """-X lazy_imports=none should be rejected.""" result = subprocess.run( - [sys.executable, "-X", "lazy_imports=none", "-c", code], + [sys.executable, "-X", "lazy_imports=none", "-c", "pass"], capture_output=True, text=True ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("EAGER", result.stdout) + self.assertNotEqual(result.returncode, 0) + self.assertIn("-X lazy_imports: invalid value", result.stderr) + self.assertIn("expected 'all' or 'normal'", result.stderr) def test_cli_lazy_imports_normal_respects_lazy_keyword_only(self): """-X lazy_imports=normal should respect lazy keyword only.""" @@ -1081,77 +1166,27 @@ def test_env_var_lazy_imports_all_enables_global_lazy(self): self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") self.assertIn("LAZY", result.stdout) - def test_env_var_lazy_imports_none_disables_all_lazy(self): - """PYTHON_LAZY_IMPORTS=none should disable all lazy imports.""" - code = textwrap.dedent(""" - import sys - lazy import json - if 'json' in sys.modules: - print("EAGER") - else: - print("LAZY") - """) + def test_env_var_lazy_imports_none_is_rejected(self): + """PYTHON_LAZY_IMPORTS=none should be rejected.""" import os env = os.environ.copy() env["PYTHON_LAZY_IMPORTS"] = "none" result = subprocess.run( - [sys.executable, "-c", code], + [sys.executable, "-c", "pass"], capture_output=True, text=True, env=env ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("EAGER", result.stdout) - - def test_cli_lazy_imports_none_disables_dunder_lazy_modules(self): - """-X lazy_imports=none should override __lazy_modules__.""" - code = textwrap.dedent(""" - import sys - __lazy_modules__ = ["json"] - import json - if 'json' in sys.modules: - print("EAGER") - else: - print("LAZY") - """) - result = subprocess.run( - [sys.executable, "-X", "lazy_imports=none", "-c", code], - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("EAGER", result.stdout) - - def test_env_var_lazy_imports_none_disables_dunder_lazy_modules(self): - """PYTHON_LAZY_IMPORTS=none should override __lazy_modules__.""" - code = textwrap.dedent(""" - import sys - __lazy_modules__ = ["json"] - import json - if 'json' in sys.modules: - print("EAGER") - else: - print("LAZY") - """) - import os - - env = os.environ.copy() - env["PYTHON_LAZY_IMPORTS"] = "none" - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True, - env=env, - ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("EAGER", result.stdout) + self.assertNotEqual(result.returncode, 0) + self.assertIn("PYTHON_LAZY_IMPORTS: invalid value", result.stderr) + self.assertIn("expected 'all' or 'normal'", result.stderr) def test_cli_overrides_env_var(self): """Command-line option should take precedence over environment variable.""" # PEP 810: -X lazy_imports takes precedence over PYTHON_LAZY_IMPORTS code = textwrap.dedent(""" import sys - lazy import json + import json if 'json' in sys.modules: print("EAGER") else: @@ -1159,23 +1194,23 @@ def test_cli_overrides_env_var(self): """) import os env = os.environ.copy() - env["PYTHON_LAZY_IMPORTS"] = "all" # env says all + env["PYTHON_LAZY_IMPORTS"] = "all" # env says all imports are lazy result = subprocess.run( - [sys.executable, "-X", "lazy_imports=none", "-c", code], # CLI says none + [sys.executable, "-X", "lazy_imports=normal", "-c", code], capture_output=True, text=True, env=env ) self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - # CLI should win - imports should be eager + # CLI should win, so a regular import should stay eager. self.assertIn("EAGER", result.stdout) def test_sys_set_lazy_imports_overrides_cli(self): """sys.set_lazy_imports() should take precedence over CLI option.""" code = textwrap.dedent(""" import sys - sys.set_lazy_imports("none") # Override CLI - lazy import json + sys.set_lazy_imports("normal") # Override CLI + import json if 'json' in sys.modules: print("EAGER") else: @@ -1191,20 +1226,12 @@ def test_sys_set_lazy_imports_overrides_cli(self): @support.requires_subprocess() -class FilterFunctionSignatureTests(unittest.TestCase): +class FilterFunctionSignatureTests(LazyImportTestCase): """Tests for the filter function signature per PEP 810. PEP 810: func(importer: str, name: str, fromlist: tuple[str, ...] | None) -> bool """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def _run_subprocess_with_modules(self, code, files): with tempfile.TemporaryDirectory() as tmpdir: for relpath, contents in files.items(): @@ -1474,17 +1501,9 @@ def my_filter(importer, name, fromlist): self.assertIn("OK", result.stdout) -class AdditionalSyntaxRestrictionTests(unittest.TestCase): +class AdditionalSyntaxRestrictionTests(LazyImportTestCase): """Additional syntax restriction tests per PEP 810.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_lazy_import_inside_class_raises_syntax_error(self): """lazy import inside class body should raise SyntaxError.""" # PEP 810: "The soft keyword is only allowed at the global (module) level, @@ -1494,7 +1513,7 @@ def test_lazy_import_inside_class_raises_syntax_error(self): @support.requires_subprocess() -class MixedLazyEagerImportTests(unittest.TestCase): +class MixedLazyEagerImportTests(LazyImportTestCase): """Tests for mixing lazy and eager imports of the same module. PEP 810: "If module foo is imported both lazily and eagerly in the same @@ -1502,14 +1521,6 @@ class MixedLazyEagerImportTests(unittest.TestCase): the same module object." """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_eager_import_before_lazy_resolves_to_same_module(self): """Eager import before lazy should make lazy resolve to same module.""" code = textwrap.dedent(""" @@ -1555,17 +1566,9 @@ def test_lazy_import_before_eager_resolves_to_same_module(self): self.assertIn("OK", result.stdout) -class RelativeImportTests(unittest.TestCase): +class RelativeImportTests(LazyImportTestCase): """Tests for relative imports with lazy keyword.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_relative_lazy_import(self): """lazy from . import submodule should work.""" from test.test_lazy_import.data import relative_lazy @@ -1590,21 +1593,13 @@ def test_relative_lazy_from_import(self): self.assertIn("test.test_lazy_import.data.basic2", sys.modules) -class LazyModulesCompatibilityFromImportTests(unittest.TestCase): +class LazyModulesCompatibilityFromImportTests(LazyImportTestCase): """Tests for __lazy_modules__ with from imports. PEP 810: "When a module is made lazy this way, from-imports using that module are also lazy" """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_lazy_modules_makes_from_imports_lazy(self): """__lazy_modules__ should make from imports of listed modules lazy.""" from test.test_lazy_import.data import lazy_compat_from @@ -1619,7 +1614,7 @@ def test_lazy_modules_makes_from_imports_lazy(self): @support.requires_subprocess() -class ImportStateAtReificationTests(unittest.TestCase): +class ImportStateAtReificationTests(LazyImportTestCase): """Tests for import system state at reification time. PEP 810: "Reification still calls __import__ to resolve the import, which uses @@ -1628,14 +1623,6 @@ class ImportStateAtReificationTests(unittest.TestCase): statement was evaluated." """ - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_sys_path_at_reification_time_is_used(self): """sys.path changes after lazy import should affect reification.""" code = textwrap.dedent(""" @@ -1678,17 +1665,9 @@ def test_sys_path_at_reification_time_is_used(self): @support.requires_subprocess() -class ThreadSafetyTests(unittest.TestCase): +class ThreadSafetyTests(LazyImportTestCase): """Tests for thread-safety of lazy imports.""" - def tearDown(self): - for key in list(sys.modules.keys()): - if key.startswith('test.test_lazy_import.data'): - del sys.modules[key] - - sys.set_lazy_imports_filter(None) - sys.set_lazy_imports("normal") - def test_concurrent_lazy_import_reification(self): """Multiple threads racing to reify the same lazy import should succeed.""" from test.test_lazy_import.data import basic_unused @@ -1804,7 +1783,7 @@ def create_lazy_imports(idx): t.join() assert not errors, f"Errors: {errors}" - assert isinstance(sys.lazy_modules, dict), "sys.lazy_modules is not a dict" + assert isinstance(sys.lazy_modules, set), "sys.lazy_modules is not a dict" print("OK") """) @@ -1950,16 +1929,13 @@ def test_normal_import_dis(self): @unittest.skipIf(_testcapi is None, 'need the _testcapi module') -class LazyCApiTests(unittest.TestCase): - def tearDown(self): - sys.set_lazy_imports("normal") - sys.set_lazy_imports_filter(None) - +class LazyCApiTests(LazyImportTestCase): def test_set_matches_sys(self): self.assertEqual(_testcapi.PyImport_GetLazyImportsMode(), sys.get_lazy_imports()) - for mode in ("normal", "all", "none"): + for mode in ("normal", "all"): _testcapi.PyImport_SetLazyImportsMode(mode) self.assertEqual(_testcapi.PyImport_GetLazyImportsMode(), sys.get_lazy_imports()) + self.assertRaises(ValueError, _testcapi.PyImport_SetLazyImportsMode, "none") def test_filter_matches_sys(self): self.assertEqual(_testcapi.PyImport_GetLazyImportsFilter(), sys.get_lazy_imports_filter()) @@ -1973,6 +1949,147 @@ def filter(*args): def test_set_bad_filter(self): self.assertRaises(ValueError, _testcapi.PyImport_SetLazyImportsFilter, 42) + def test_dunder_lazy_import_without_frame(self): + # gh-151510: __lazy_import__() called with no globals and no running + # Python frame must raise TypeError instead of crashing. + with self.assertRaisesRegex( + TypeError, + r"__lazy_import__\(\) missing globals when called without a frame", + ): + _testcapi.lazy_import_without_frame( + "test.test_lazy_import.data.basic2" + ) + + +class SubmoduleLazinessTests(unittest.TestCase): + """Tests that module-level lazy imports remain lazy until accessed.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_unaccessed_imports_stay_lazy(self): + """Imports in 'all' mode should stay lazy until accessed.""" + sys.set_lazy_imports("all") + from test.test_lazy_import.data.metasyntactic import names + self.assertIsInstance(names.__dict__["Foo"], types.LazyImportType) + self.assertNotIn( + "test.test_lazy_import.data.metasyntactic.foo", sys.modules + ) + _ = names.Foo + self.assertEqual(names.Foo, "Foo") + self.assertIn( + "test.test_lazy_import.data.metasyntactic.foo", sys.modules + ) + self.assertIsInstance(names.__dict__["Ack"], types.LazyImportType) + self.assertNotIn( + "test.test_lazy_import.data.metasyntactic.foo.ack", sys.modules + ) + + +class AttributeSideEffectTests(unittest.TestCase): + """Tests that submodule imports don't overwrite parent attributes.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_version_submodule_does_not_overwrite(self): + """A __version__ submodule should not overwrite the parent's + __version__ attribute imported in __init__.py.""" + import test.test_lazy_import.data.versioned as versioned + self.assertEqual(versioned.__version__, "1.0") + self.assertEqual( + versioned.__copyright__, + "Copyright (c) 2001-2022 Python Software Foundation.", + ) + + +class ModuleVariableNameCollisionTests(unittest.TestCase): + """Tests for name collision between a submodule and a variable.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_variable_after_import_wins(self): + """Variable assigned after import should overwrite the submodule.""" + from test.test_lazy_import.data import module_same_name_var_order1 + self.assertEqual(module_same_name_var_order1.bar, "Blah") + + def test_import_after_variable_wins(self): + """Import after variable assignment should overwrite the variable.""" + from test.test_lazy_import.data import module_same_name_var_order2 + bar_mod = sys.modules[ + "test.test_lazy_import.data.module_same_name_var_order2.bar" + ] + self.assertIs(module_same_name_var_order2.bar, bar_mod) + + +class DeletedModuleReimportTests(unittest.TestCase): + """Tests for reimporting after module deletion from sys.modules.""" + + def tearDown(self): + for key in list(sys.modules.keys()): + if key.startswith('test.test_lazy_import.data'): + del sys.modules[key] + sys.set_lazy_imports_filter(None) + sys.set_lazy_imports("normal") + + def test_reimport_creates_new_module(self): + """Deleting and reimporting should create a new module object.""" + import test.test_lazy_import.data.metasyntactic.foo + import test.test_lazy_import.data.metasyntactic.foo.bar.baz + + first_bar = test.test_lazy_import.data.metasyntactic.foo.bar + + del sys.modules[ + "test.test_lazy_import.data.metasyntactic.foo.bar" + ] + + import test.test_lazy_import.data.metasyntactic.foo.bar.thud + + second_bar = test.test_lazy_import.data.metasyntactic.foo.bar + + self.assertIsNot(first_bar, second_bar) + self.assertIn("baz", dir(first_bar)) + self.assertNotIn("thud", dir(first_bar)) + self.assertIn("thud", dir(second_bar)) + self.assertNotIn("baz", dir(second_bar)) + + +@support.requires_subprocess() +class CircularImportLazyTests(unittest.TestCase): + """Tests that lazy imports can break circular import patterns.""" + + def test_succeeds_with_lazy(self): + """Same-level circular imports should succeed with lazy mode.""" + proc = assert_python_ok( + "-X", "lazy_imports=all", "-c", + "import test.test_lazy_import.data.circular_import_pkg.main;" + "print('OK')", + ) + self.assertIn(b"OK", proc.out) + + def test_fails_without_lazy(self): + """Same-level circular imports should fail without lazy mode.""" + result = subprocess.run( + [sys.executable, "-c", + "import test.test_lazy_import.data.circular_import_pkg.main"], + capture_output=True, text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("ImportError", result.stderr) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_lazy_import/__main__.py b/Lib/test/test_lazy_import/__main__.py new file mode 100644 index 000000000000000..d6c94efaf30833e --- /dev/null +++ b/Lib/test/test_lazy_import/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_lazy_import') diff --git a/Lib/test/test_lazy_import/data/circular_import_pkg/__init__.py b/Lib/test/test_lazy_import/data/circular_import_pkg/__init__.py new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/Lib/test/test_lazy_import/data/circular_import_pkg/main.py b/Lib/test/test_lazy_import/data/circular_import_pkg/main.py new file mode 100644 index 000000000000000..0b897b377e28a66 --- /dev/null +++ b/Lib/test/test_lazy_import/data/circular_import_pkg/main.py @@ -0,0 +1,2 @@ +from .x import X2 +X2() diff --git a/Lib/test/test_lazy_import/data/circular_import_pkg/x.py b/Lib/test/test_lazy_import/data/circular_import_pkg/x.py new file mode 100644 index 000000000000000..b44d5185c9e70fc --- /dev/null +++ b/Lib/test/test_lazy_import/data/circular_import_pkg/x.py @@ -0,0 +1,7 @@ +def X1(): + return "X" + +from .y import Y1 + +def X2(): + return Y1() diff --git a/Lib/test/test_lazy_import/data/circular_import_pkg/y.py b/Lib/test/test_lazy_import/data/circular_import_pkg/y.py new file mode 100644 index 000000000000000..d2385f7e67c01fd --- /dev/null +++ b/Lib/test/test_lazy_import/data/circular_import_pkg/y.py @@ -0,0 +1,7 @@ +def Y1(): + return "Y" + +from .x import X2 + +def Y2(): + return X2() diff --git a/Lib/test/test_lazy_import/data/global_off.py b/Lib/test/test_lazy_import/data/global_off.py deleted file mode 100644 index 95d1511dd932232..000000000000000 --- a/Lib/test/test_lazy_import/data/global_off.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys - -sys.set_lazy_imports("none") - -lazy import test.test_lazy_import.data.basic2 as basic2 diff --git a/Lib/test/test_lazy_import/data/metasyntactic/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/__init__.py new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/__init__.py new file mode 100644 index 000000000000000..632a9fb87ac0fd7 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/__init__.py @@ -0,0 +1 @@ +Foo = "Foo" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/ack/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/ack/__init__.py new file mode 100644 index 000000000000000..c4c249f3b651536 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/ack/__init__.py @@ -0,0 +1 @@ +Ack = "Ack" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/__init__.py new file mode 100644 index 000000000000000..03faa550258f97f --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/__init__.py @@ -0,0 +1 @@ +Bar = "Bar" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/__init__.py new file mode 100644 index 000000000000000..62d8366b4404d6c --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/__init__.py @@ -0,0 +1 @@ +Baz = "Baz" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/qux/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/qux/__init__.py new file mode 100644 index 000000000000000..12389d76a28539c --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/baz/qux/__init__.py @@ -0,0 +1 @@ +Qux = "Qux" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/thud/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/thud/__init__.py new file mode 100644 index 000000000000000..a4dd8ab0fae17ee --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/foo/bar/thud/__init__.py @@ -0,0 +1 @@ +Thud = "Thud" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/names.py b/Lib/test/test_lazy_import/data/metasyntactic/names.py new file mode 100644 index 000000000000000..b2edf13e850389a --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/names.py @@ -0,0 +1,9 @@ +from .foo import Foo +from .foo.ack import Ack +from .foo.bar import Bar +from .foo.bar.baz import Baz +from .foo.bar.thud import Thud +from .waldo import Waldo +from .waldo.fred import Fred +from .plugh import Plugh +Metasyntactic = "Metasyntactic" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/plugh/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/plugh/__init__.py new file mode 100644 index 000000000000000..5db053204fc17ce --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/plugh/__init__.py @@ -0,0 +1 @@ +Plugh = "Plugh" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/waldo/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/waldo/__init__.py new file mode 100644 index 000000000000000..39f5a1b18385733 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/waldo/__init__.py @@ -0,0 +1 @@ +Waldo = "Waldo" diff --git a/Lib/test/test_lazy_import/data/metasyntactic/waldo/fred/__init__.py b/Lib/test/test_lazy_import/data/metasyntactic/waldo/fred/__init__.py new file mode 100644 index 000000000000000..8c9dce1b3cfb9d1 --- /dev/null +++ b/Lib/test/test_lazy_import/data/metasyntactic/waldo/fred/__init__.py @@ -0,0 +1 @@ +Fred = "Fred" diff --git a/Lib/test/test_lazy_import/data/module_same_name_var_order1/__init__.py b/Lib/test/test_lazy_import/data/module_same_name_var_order1/__init__.py new file mode 100644 index 000000000000000..b815b2a7d6fedf1 --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_same_name_var_order1/__init__.py @@ -0,0 +1,3 @@ +from .bar import Bar +bar = "Blah" +Bar diff --git a/Lib/test/test_lazy_import/data/module_same_name_var_order1/bar.py b/Lib/test/test_lazy_import/data/module_same_name_var_order1/bar.py new file mode 100644 index 000000000000000..03faa550258f97f --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_same_name_var_order1/bar.py @@ -0,0 +1 @@ +Bar = "Bar" diff --git a/Lib/test/test_lazy_import/data/module_same_name_var_order2/__init__.py b/Lib/test/test_lazy_import/data/module_same_name_var_order2/__init__.py new file mode 100644 index 000000000000000..008d199c73e534d --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_same_name_var_order2/__init__.py @@ -0,0 +1,3 @@ +bar = "Blah" +from .bar import Bar +Bar diff --git a/Lib/test/test_lazy_import/data/module_same_name_var_order2/bar.py b/Lib/test/test_lazy_import/data/module_same_name_var_order2/bar.py new file mode 100644 index 000000000000000..03faa550258f97f --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_same_name_var_order2/bar.py @@ -0,0 +1 @@ +Bar = "Bar" diff --git a/Lib/test/test_lazy_import/data/module_with_getattr.py b/Lib/test/test_lazy_import/data/module_with_getattr.py new file mode 100644 index 000000000000000..db3a2301075c2ee --- /dev/null +++ b/Lib/test/test_lazy_import/data/module_with_getattr.py @@ -0,0 +1,12 @@ +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" + raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/pkg/__init__.py b/Lib/test/test_lazy_import/data/pkg/__init__.py index 2d76abaa89f8937..5f7b8662596cac6 100644 --- a/Lib/test/test_lazy_import/data/pkg/__init__.py +++ b/Lib/test/test_lazy_import/data/pkg/__init__.py @@ -1 +1,14 @@ x = 42 + +def __getattr__(name): + if name == "dynamic_attr": + return "from_getattr" + elif name == "raising_attr": + raise ValueError("from_getattr") + elif name == "import_error_attr": + raise ImportError(name) + elif name == "warning_attr": + import warnings + warnings.warn("from_getattr", category=UserWarning) + return "from_warning_attr" + raise AttributeError(name) diff --git a/Lib/test/test_lazy_import/data/versioned/__init__.py b/Lib/test/test_lazy_import/data/versioned/__init__.py new file mode 100644 index 000000000000000..be5bb3f8e607185 --- /dev/null +++ b/Lib/test/test_lazy_import/data/versioned/__init__.py @@ -0,0 +1,2 @@ +from .__version__ import __version__ +from .__version__ import __copyright__ diff --git a/Lib/test/test_lazy_import/data/versioned/__version__.py b/Lib/test/test_lazy_import/data/versioned/__version__.py new file mode 100644 index 000000000000000..37a104387354e28 --- /dev/null +++ b/Lib/test/test_lazy_import/data/versioned/__version__.py @@ -0,0 +1,2 @@ +__version__ = "1.0" +__copyright__ = "Copyright (c) 2001-2022 Python Software Foundation." diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index cee528722b85aa0..cf3796d94808015 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -171,6 +171,17 @@ def test_references___class__(self): """ self._check_in_scopes(code, raises=NameError) + def test_references___class___nested(self): + code = """ + res = [(lambda: __class__)() for _ in [1]] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___class___nested_used(self): + class _C: + res = [lambda: __class__ for _ in [1]] + self.assertIs(_C.res[0](), _C) + def test_references___class___defined(self): code = """ __class__ = 2 @@ -180,18 +191,38 @@ def test_references___class___defined(self): code, outputs={"res": [2]}, scopes=["module", "function"]) self._check_in_scopes(code, raises=NameError, scopes=["class"]) + def test_references___class___defined_nested(self): + code = """ + __class__ = 2 + res = [(lambda: __class__)() for x in [1]] + """ + self._check_in_scopes( + code, outputs={"res": [2]}, scopes=["module", "function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + def test_references___classdict__(self): code = """ class i: [__classdict__ for x in y] """ self._check_in_scopes(code, raises=NameError) + def test_references___classdict___nested(self): + class _C: + res = [(lambda: __classdict__)() for _ in [1]] + self.assertIn("res", _C.res[0]) + def test_references___conditional_annotations__(self): code = """ class i: [__conditional_annotations__ for x in y] """ self._check_in_scopes(code, raises=NameError) + def test_references___conditional_annotations___nested(self): + code = """ + class i: [lambda: __conditional_annotations__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + def test_references___class___enclosing(self): code = """ __class__ = 2 diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index a06c600cf56689e..8057c35f540841b 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -9,6 +9,15 @@ import sys import codecs + +class MiscTestCase(unittest.TestCase): + maxDiff = None + def test__all__(self): + extra = ["localeconv", "strcoll", "strxfrm", "getencoding", + "Error"] + not_exported = ["locale_encoding_alias", "locale_alias", "windows_locale"] + support.check__all__(self, locale, extra=extra, not_exported=not_exported) + class LazyImportTest(unittest.TestCase): @cpython_only def test_lazy_import(self): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 1a76c2173a30117..fcd3da61a078aec 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -3297,12 +3297,11 @@ def format(self, record): } } - # Remove when deprecation ends. - class DeprecatedStrmHandler(logging.StreamHandler): + class StrmHandler(logging.StreamHandler): def __init__(self, strm=None): super().__init__(stream=strm) - config_custom_handler_with_deprecated_strm_arg = { + config_custom_handler_with_removed_strm_arg = { "version": 1, "formatters": { "form1": { @@ -3311,7 +3310,7 @@ def __init__(self, strm=None): }, "handlers": { "hand1": { - "class": DeprecatedStrmHandler, + "class": StrmHandler, "formatter": "form1", "level": "NOTSET", "stream": "ext://sys.stdout", @@ -3417,14 +3416,9 @@ def test_config5_ok(self): self.test_config1_ok(config=self.config5) self.check_handler('hand1', CustomHandler) - def test_deprecation_warning_custom_handler_with_strm_arg(self): - msg = ( - "Support for custom logging handlers with the 'strm' argument " - "is deprecated and scheduled for removal in Python 3.16. " - "Define handlers with the 'stream' argument instead." - ) - with self.assertWarnsRegex(DeprecationWarning, msg): - self.test_config1_ok(config=self.config_custom_handler_with_deprecated_strm_arg) + def test_removed_strm_arg(self): + with self.assertRaisesRegex(ValueError, 'hand1'): + self.apply_config(self.config_custom_handler_with_removed_strm_arg) def test_config6_failure(self): self.assertRaises(Exception, self.apply_config, self.config6) @@ -4071,11 +4065,7 @@ def test_config_reject_simple_queue_handler_multiprocessing_context(self): # and thus cannot be used as a queue-like object (gh-124653) import multiprocessing - - if support.MS_WINDOWS: - start_methods = ['spawn'] - else: - start_methods = ['spawn', 'fork', 'forkserver'] + start_methods = multiprocessing.get_all_start_methods() for start_method in start_methods: with self.subTest(start_method=start_method): @@ -4091,10 +4081,8 @@ def test_config_reject_simple_queue_handler_multiprocessing_context(self): " assertions in multiprocessing") def test_config_queue_handler_multiprocessing_context(self): # regression test for gh-121723 - if support.MS_WINDOWS: - start_methods = ['spawn'] - else: - start_methods = ['spawn', 'fork', 'forkserver'] + import multiprocessing + start_methods = multiprocessing.get_all_start_methods() for start_method in start_methods: with self.subTest(start_method=start_method): ctx = multiprocessing.get_context(start_method) @@ -4185,6 +4173,30 @@ def test_90195(self): # Logger should be enabled, since explicitly mentioned self.assertFalse(logger.disabled) + def test_disable_existing_loggers_preserves_children(self): + parent = logging.getLogger('many') + child = logging.getLogger('many.child') + child.setLevel(logging.CRITICAL) + self.assertFalse(child.isEnabledFor(logging.INFO)) + cousin = logging.getLogger('many-child') + for i in range(20): + logging.getLogger(f'many-sibling-{i}') + + self.apply_config({ + 'version': 1, + 'loggers': { + 'many': { + 'level': 'INFO', + }, + }, + }) + + self.assertFalse(parent.disabled) + self.assertFalse(child.disabled) + self.assertEqual(child.level, logging.NOTSET) + self.assertTrue(child.isEnabledFor(logging.INFO)) + self.assertTrue(cousin.disabled) + def test_111615(self): # See gh-111615 import_helper.import_module('_multiprocessing') # see gh-113692 @@ -4257,6 +4269,43 @@ def test_set_log_record_factory(self): man.setLogRecordFactory(expected) self.assertEqual(man.logRecordFactory, expected) + @threading_helper.requires_working_threading() + def test_getLogger_fast_path_never_returns_unwired_logger(self): + # getLogger()'s lock-free fast path returns a logger straight out of + # loggerDict, so a logger must be published there only after + # _fixupParents() has set its parent; otherwise a concurrent caller + # observes it detached from the hierarchy (gh-150818 follow-up). + manager = logging.Manager(logging.RootLogger(logging.WARNING)) + name = 'a.b.c' + + paused = threading.Event() + seen = [] + real_fixup = manager._fixupParents + + # Pause the creating thread between publishing rv and wiring its + # parent, then read loggerDict the way the fast path does and snapshot + # the parent at that instant (rv is wired in place soon after). + def fixup(alogger): + paused.set() + reader.join() + real_fixup(alogger) + + def read(): + paused.wait() + rv = manager.loggerDict.get(name) + if rv is not None and not isinstance(rv, logging.PlaceHolder): + seen.append(rv.parent) + + reader = threading.Thread(target=read) + manager._fixupParents = fixup + try: + reader.start() + manager.getLogger(name) + finally: + manager._fixupParents = real_fixup + + self.assertNotIn(None, seen) + class ChildLoggerTest(BaseTest): def test_child_loggers(self): r = logging.getLogger() @@ -6603,22 +6652,71 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) + @unittest.skipUnless(hasattr(os.stat_result, 'st_birthtime') or hasattr(os, 'statx'), + "st_birthtime and statx() not available or supported by Python on this OS") + @support.requires_resource('walltime') + def test_rollover_based_on_st_birthtime_only(self): + def add_record(message: str) -> None: + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, when='S', interval=4, encoding="utf-8", backupCount=1) + fmt = logging.Formatter('%(asctime)s %(message)s') + fh.setFormatter(fmt) + record = logging.makeLogRecord({'msg': message}) + fh.emit(record) + fh.close() + + add_record('testing - initial') + self.assertLogFile(self.fn) + # Sleep a little over the half of rollover time - and this value + # must be over 2 seconds, since this is the mtime resolution on + # FAT32 filesystems. + time.sleep(2.1) + add_record('testing - update before rollover to renew the st_mtime') + time.sleep(2.1) # a little over the half of rollover time + add_record('testing - new record supposedly in the new file after rollover') + + # At this point, the log file should be rotated if the rotation + # is based on creation time but should be not if it's based on + # modification time. + found = False + now = datetime.datetime.now() + GO_BACK = 5 # seconds + for secs in range(GO_BACK + 1): + prev = now - datetime.timedelta(seconds=secs) + fn = self.fn + prev.strftime(".%Y-%m-%d_%H-%M-%S") + found = os.path.exists(fn) + if found: + self.rmfiles.append(fn) + break + msg = 'No rotated files found, went back %d seconds' % GO_BACK + if not found: + # print additional diagnostics + dn, fn = os.path.split(self.fn) + files = [f for f in os.listdir(dn) if f.startswith(fn)] + print('Test time: %s' % now.strftime("%Y-%m-%d %H-%M-%S"), file=sys.stderr) + print('The only matching files are: %s' % files, file=sys.stderr) + for f in files: + print('Contents of %s:' % f) + path = os.path.join(dn, f) + print(os.stat(path)) + with open(path, 'r') as tf: + print(tf.read()) + self.assertTrue(found, msg=msg) + def test_rollover_at_midnight(self, weekly=False): os_helper.unlink(self.fn) + # Emit the first records a little after the beginning of a whole + # second, so that their file times fall inside that second and not the + # previous one, which would cause an unwanted rollover. now = datetime.datetime.now() - atTime = now.time() - if not 0.1 < atTime.microsecond/1e6 < 0.9: - # The test requires all records to be emitted within - # the range of the same whole second. - time.sleep((0.1 - atTime.microsecond/1e6) % 1.0) + if not 0.1 < now.microsecond/1e6 < 0.9: + time.sleep((0.1 - now.microsecond/1e6) % 1.0) now = datetime.datetime.now() - atTime = now.time() - atTime = atTime.replace(microsecond=0) fmt = logging.Formatter('%(asctime)s %(message)s') when = f'W{now.weekday()}' if weekly else 'MIDNIGHT' for i in range(3): fh = logging.handlers.TimedRotatingFileHandler( - self.fn, encoding="utf-8", when=when, atTime=atTime) + self.fn, encoding="utf-8", when=when, atTime=now.time()) fh.setFormatter(fmt) r2 = logging.makeLogRecord({'msg': f'testing1 {i}'}) fh.emit(r2) @@ -6628,7 +6726,13 @@ def test_rollover_at_midnight(self, weekly=False): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) - os.utime(self.fn, (now.timestamp() - 1,)*2) + # The creation time, used to compute the rollover time, cannot be + # changed, so the rollover cannot be forced by back-dating the file. + # Wait until the clock reaches a rollover time set one second ahead. + rollover = int(time.time()) + 1 + atTime = datetime.datetime.fromtimestamp(rollover).time() + while time.time() < rollover: + time.sleep(rollover - time.time()) for i in range(2): fh = logging.handlers.TimedRotatingFileHandler( self.fn, encoding="utf-8", when=when, atTime=atTime) @@ -6636,7 +6740,8 @@ def test_rollover_at_midnight(self, weekly=False): r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) fh.emit(r2) fh.close() - rolloverDate = now - datetime.timedelta(days=7 if weekly else 1) + rolloverDate = (datetime.datetime.fromtimestamp(rollover) + - datetime.timedelta(days=7 if weekly else 1)) otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' self.assertLogFile(otherfn) with open(self.fn, encoding="utf-8") as f: diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index e93c3c37354e27f..d4f954b34c12526 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -383,6 +383,12 @@ def test_uninitialized_LZMADecompressor_crash(self): self.assertEqual(LZMADecompressor.__new__(LZMADecompressor). decompress(bytes()), b'') + def test_riscv_filter_constant_exists(self): + self.assertTrue(lzma.FILTER_RISCV) + + def test_arm64_filter_constant_exists(self): + self.assertTrue(lzma.FILTER_ARM64) + class CompressDecompressFunctionTestCase(unittest.TestCase): diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 8f9a239bead1309..cc37697189af3e5 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1,7 +1,8 @@ # Python test set -- math module # XXXX Should not do tests around zero only -from test.support import verbose, requires_IEEE_754 +from test.support import (verbose, requires_IEEE_754, + skip_if_double_rounding) from test import support import unittest import fractions @@ -23,11 +24,6 @@ FLOAT_MAX = sys.float_info.max FLOAT_MIN = sys.float_info.min -# detect evidence of double-rounding: fsum is not always correctly -# rounded on machines that suffer from double rounding. -x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer -HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) - # locate file with test values if __name__ == '__main__': file = sys.argv[0] @@ -79,7 +75,7 @@ def parse_mtestfile(fname): -- starts a comment blank lines, or lines containing only a comment, are ignored other lines are expected to have the form - id fn arg -> expected [flag]* + id fn arg... -> expected [flag]* """ with open(fname, encoding="utf-8") as fp: @@ -91,12 +87,12 @@ def parse_mtestfile(fname): continue lhs, rhs = line.split('->') - id, fn, arg = lhs.split() + id, fn, *args = lhs.split() rhs_pieces = rhs.split() exp = rhs_pieces[0] flags = rhs_pieces[1:] - yield (id, fn, float(arg), float(exp), flags) + yield (id, fn, [float(arg) for arg in args], float(exp), flags) def parse_testfile(fname): @@ -683,8 +679,7 @@ def testfrexp(name, result, expected): self.assertTrue(math.isnan(math.frexp(NAN)[0])) @requires_IEEE_754 - @unittest.skipIf(HAVE_DOUBLE_ROUNDING, - "fsum is not exact on machines with double rounding") + @skip_if_double_rounding def testFsum(self): # math.fsum relies on exact rounding for correct operation. # There's a known problem with IA32 floating-point that causes @@ -920,8 +915,8 @@ def testHypot(self): self.assertRaises(TypeError, math.hypot, *([1.0]*18), 'spam') @requires_IEEE_754 - @unittest.skipIf(HAVE_DOUBLE_ROUNDING, - "hypot() loses accuracy on machines with double rounding") + @skip_if_double_rounding + @support.skip_on_newlib def testHypotAccuracy(self): # Verify improved accuracy in cases that were known to be inaccurate. # @@ -1253,12 +1248,6 @@ def testLog2(self): self.assertEqual(math.log2(4), 2.0) self.assertEqual(math.log2(MyIndexable(4)), 2.0) - # Large integer values - self.assertEqual(math.log2(2**1023), 1023.0) - self.assertEqual(math.log2(2**1024), 1024.0) - self.assertEqual(math.log2(2**2000), 2000.0) - self.assertEqual(math.log2(MyIndexable(2**2000)), 2000.0) - self.assertRaises(ValueError, math.log2, 0.0) self.assertRaises(ValueError, math.log2, 0) self.assertRaises(ValueError, math.log2, MyIndexable(0)) @@ -1276,12 +1265,19 @@ def testLog2(self): @requires_IEEE_754 # log2() is not accurate enough on Mac OS X Tiger (10.4) @support.requires_mac_ver(10, 5) + @support.skip_on_newlib def testLog2Exact(self): # Check that we get exact equality for log2 of powers of 2. actual = [math.log2(math.ldexp(1.0, n)) for n in range(-1074, 1024)] expected = [float(n) for n in range(-1074, 1024)] self.assertEqual(actual, expected) + # Large integer values + self.assertEqual(math.log2(2**1023), 1023.0) + self.assertEqual(math.log2(2**1024), 1024.0) + self.assertEqual(math.log2(2**2000), 2000.0) + self.assertEqual(math.log2(MyIndexable(2**2000)), 2000.0) + def testLog10(self): self.assertRaises(TypeError, math.log10) self.ftest('log10(0.1)', math.log10(0.1), -1) @@ -1410,8 +1406,7 @@ def __rmul__(self, other): self.assertEqual(sumprod(*args), 0.0) @requires_IEEE_754 - @unittest.skipIf(HAVE_DOUBLE_ROUNDING, - "sumprod() accuracy not guaranteed on machines with double rounding") + @skip_if_double_rounding @support.cpython_only # Other implementations may choose a different algorithm def test_sumprod_accuracy(self): sumprod = math.sumprod @@ -1496,8 +1491,7 @@ def run(func, *args): ) @requires_IEEE_754 - @unittest.skipIf(HAVE_DOUBLE_ROUNDING, - "sumprod() accuracy not guaranteed on machines with double rounding") + @skip_if_double_rounding @support.cpython_only # Other implementations may choose a different algorithm @support.requires_resource('cpu') def test_sumprod_extended_precision_accuracy(self): @@ -2147,10 +2141,10 @@ def test_testfile(self): @requires_IEEE_754 def test_mtestfile(self): - fail_fmt = "{}: {}({!r}): {}" + fail_fmt = "{}: {}{!r}: {}" failures = [] - for id, fn, arg, expected, flags in parse_mtestfile(math_testcases): + for id, fn, args, expected, flags in parse_mtestfile(math_testcases): func = getattr(math, fn) if 'invalid' in flags or 'divide-by-zero' in flags: @@ -2159,7 +2153,7 @@ def test_mtestfile(self): expected = 'OverflowError' try: - got = func(arg) + got = func(*args) except ValueError: got = 'ValueError' except OverflowError: @@ -2182,7 +2176,7 @@ def test_mtestfile(self): # general. abs_tol = 1e-15 - elif fn == 'erfc' and arg >= 0.0: + elif fn == 'erfc' and (arg := args[0]) >= 0.0: # erfc has less-than-ideal accuracy for large # arguments (x ~ 25 or so), mainly due to the # error involved in computing exp(-x*x). @@ -2205,7 +2199,7 @@ def test_mtestfile(self): if failure is None: continue - msg = fail_fmt.format(id, fn, arg, failure) + msg = fail_fmt.format(id, fn, args, failure) failures.append(msg) if failures: @@ -2615,6 +2609,7 @@ def test_fma_nan_results(self): self.assertIsNaN(math.fma(a, math.nan, b)) self.assertIsNaN(math.fma(a, b, math.nan)) + @support.skip_on_newlib def test_fma_infinities(self): # Cases involving infinite inputs or results. positives = [1e-300, 2.3, 1e300, math.inf] @@ -2685,7 +2680,7 @@ def test_fma_infinities(self): # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 # properly: it doesn't use the right sign when the result is zero. @unittest.skipIf( - sys.platform.startswith(("freebsd", "wasi", "netbsd", "emscripten")) + sys.platform.startswith(("freebsd", "wasi", "netbsd", "emscripten", "cygwin")) or (sys.platform == "android" and platform.machine() == "x86_64") or support.linked_to_musl(), # gh-131032 f"this platform doesn't implement IEE 754-2008 properly") @@ -2743,6 +2738,7 @@ def test_fma_zero_result(self): self.assertIsNegativeZero(math.fma(y-x, -(x+y), -z)) self.assertIsPositiveZero(math.fma(x-y, -(x+y), z)) + @support.skip_on_newlib def test_fma_overflow(self): a = b = float.fromhex('0x1p512') c = float.fromhex('0x1p1023') @@ -2776,11 +2772,13 @@ def test_fma_overflow(self): c = float.fromhex('0x1.fffffffffffffp+1023') self.assertEqual(math.fma(a, b, -c), c) + @support.skip_on_newlib def test_fma_single_round(self): a = float.fromhex('0x1p-50') self.assertEqual(math.fma(a - 1.0, a + 1.0, 1.0), a*a) - def test_random(self): + @support.skip_on_newlib + def test_fma_random(self): # A collection of randomly generated inputs for which the naive FMA # (with two rounds) gives a different result from a singly-rounded FMA. diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index f777ba583c5c069..f71b6f53486509b 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -637,7 +637,7 @@ def check_equal(view, is_equal): check_equal(m, True) # Test complex formats - for complex_format in ('F', 'D', 'Zf', 'Zd'): + for complex_format in ('Zf', 'Zd'): with self.subTest(format=complex_format): data = struct.pack(complex_format * 3, 1.0, 2.0, float('nan')) m = memoryview(data).cast(complex_format) @@ -719,10 +719,6 @@ def test_half_float(self): def test_complex_types(self): float_complex_data = struct.pack('FFF', 0.0, -1.5j, 1+2j) double_complex_data = struct.pack('DDD', 0.0, -1.5j, 1+2j) - float_complex_view = memoryview(float_complex_data).cast('F') - double_complex_view = memoryview(double_complex_data).cast('D') - self.assertEqual(float_complex_view.nbytes * 2, double_complex_view.nbytes) - self.assertListEqual(float_complex_view.tolist(), double_complex_view.tolist()) float_complex_view = memoryview(float_complex_data).cast('Zf') double_complex_view = memoryview(double_complex_data).cast('Zd') self.assertEqual(float_complex_view.nbytes * 2, double_complex_view.nbytes) diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 2d618081521e10d..19983fa3fa7628d 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -6,8 +6,9 @@ import unittest.mock from platform import win32_edition from test import support -from test.support import cpython_only, force_not_colorized, os_helper +from test.support import cpython_only, force_not_colorized, os_helper, requires_subprocess from test.support.import_helper import ensure_lazy_imports +from test.support.script_helper import assert_python_ok, assert_python_failure try: import _winapi @@ -30,50 +31,11 @@ def tearDownModule(): mimetypes.knownfiles = knownfiles -class MimeTypesTestCase(unittest.TestCase): +class MimeTypesModuleTestCase(unittest.TestCase): def setUp(self): - self.db = mimetypes.MimeTypes() - - def test_case_sensitivity(self): - eq = self.assertEqual - eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) - eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) - eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) - eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) - eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) - eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) - - def test_default_data(self): - eq = self.assertEqual - eq(self.db.guess_file_type("foo.html"), ("text/html", None)) - eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) - - def test_data_urls(self): - eq = self.assertEqual - guess_type = self.db.guess_type - eq(guess_type("data:invalidDataWithoutComma"), (None, None)) - eq(guess_type("data:,thisIsTextPlain"), ("text/plain", None)) - eq(guess_type("data:;base64,thisIsTextPlain"), ("text/plain", None)) - eq(guess_type("data:text/x-foo,thisIsTextXFoo"), ("text/x-foo", None)) - - def test_file_parsing(self): - eq = self.assertEqual - sio = io.StringIO("x-application/x-unittest pyunit\n") - self.db.readfp(sio) - eq(self.db.guess_file_type("foo.pyunit"), - ("x-application/x-unittest", None)) - eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") + mimetypes.inited = False + mimetypes._default_mime_types() + mimetypes._db = None def test_read_mime_types(self): eq = self.assertEqual @@ -108,100 +70,6 @@ def test_read_mime_types(self): mock_open.assert_called_with(filename, encoding='utf-8') eq(mime_dict[".Français"], "application/no-mans-land") - def test_non_standard_types(self): - eq = self.assertEqual - # First try strict - eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) - # And then non-strict - eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) - eq(self.db.guess_extension('image/jpeg', strict=False), '.jpg') - eq(self.db.guess_extension('image/JPEG', strict=False), '.jpg') - - def test_filename_with_url_delimiters(self): - # bpo-38449: URL delimiters cases should be handled also. - # They would have different mime types if interpreted as URL as - # compared to when interpreted as filename because of the semicolon. - eq = self.assertEqual - gzip_expected = ('application/x-tar', 'gzip') - for name in ( - ';1.tar.gz', - '?1.tar.gz', - '#1.tar.gz', - '#1#.tar.gz', - ';1#.tar.gz', - ';&1=123;?.tar.gz', - '?k1=v1&k2=v2.tar.gz', - ): - for prefix in ('', '/', '\\', - 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', - '//share/server/', '\\\\share\\server\\'): - path = prefix + name - with self.subTest(path=path): - eq(self.db.guess_file_type(path), gzip_expected) - eq(self.db.guess_type(path), gzip_expected) - expected = (None, None) if os.name == 'nt' else gzip_expected - for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): - path = prefix + name - with self.subTest(path=path): - eq(self.db.guess_file_type(path), expected) - eq(self.db.guess_type(path), expected) - eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) - eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) - - eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) - eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) - expected = (None, 'gzip') if os.name == 'nt' else gzip_expected - eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) - eq(self.db.guess_type(r'foo\.tar.gz'), expected) - eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) - - def test_url(self): - result = self.db.guess_type('http://example.com/host.html') - result = self.db.guess_type('http://host.html') - msg = 'URL only has a host name, not a file' - self.assertSequenceEqual(result, (None, None), msg) - result = self.db.guess_type('http://example.com/host.html') - msg = 'Should be text/html' - self.assertSequenceEqual(result, ('text/html', None), msg) - result = self.db.guess_type('http://example.com/host.html#x.tar') - self.assertSequenceEqual(result, ('text/html', None)) - result = self.db.guess_type('http://example.com/host.html?q=x.tar') - self.assertSequenceEqual(result, ('text/html', None)) - - def test_guess_all_types(self): - # First try strict. Use a set here for testing the results because if - # test_urllib2 is run before test_mimetypes, global state is modified - # such that the 'all' set will have more items in it. - all = self.db.guess_all_extensions('text/plain', strict=True) - self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) - self.assertEqual(len(set(all)), len(all)) # no duplicates - # And now non-strict - all = self.db.guess_all_extensions('image/jpeg', strict=False) - self.assertEqual(all, ['.jpg', '.jpe', '.jpeg']) - # And now for no hits - all = self.db.guess_all_extensions('image/jpg', strict=True) - self.assertEqual(all, []) - # And now for type existing in both strict and non-strict mappings. - self.db.add_type('test-type', '.strict-ext') - self.db.add_type('test-type', '.non-strict-ext', strict=False) - all = self.db.guess_all_extensions('test-type', strict=False) - self.assertEqual(all, ['.strict-ext', '.non-strict-ext']) - all = self.db.guess_all_extensions('test-type') - self.assertEqual(all, ['.strict-ext']) - # Test that changing the result list does not affect the global state - all.append('.no-such-ext') - all = self.db.guess_all_extensions('test-type') - self.assertNotIn('.no-such-ext', all) - - def test_encoding(self): - filename = support.findfile("mime.types") - mimes = mimetypes.MimeTypes([filename]) - exts = mimes.guess_all_extensions('application/vnd.geocube+xml', - strict=True) - self.assertEqual(exts, ['.g3', '.g\xb3']) - def test_init_reinitializes(self): # Issue 4936: make sure an init starts clean # First, put some poison into the types table @@ -340,6 +208,249 @@ def test_init_stability(self): self.assertEqual(types_map, mimetypes.types_map) self.assertEqual(common_types, mimetypes.common_types) + def test_init_files(self): + guess_file_type = mimetypes.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + mimetypes.init([filename]) + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + mimetypes.init() + self.assertEqual(guess_file_type("file.test2")[0], None) + + def test_init_knownfiles(self): + guess_file_type = mimetypes.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + mimetypes.knownfiles = [filename, "non-existent"] + self.addCleanup(mimetypes.knownfiles.clear) + + mimetypes.init() + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + def test_added_types_are_used(self): + mimetypes.add_type('testing/default-type', '') + mime_type, _ = mimetypes.guess_type('') + self.assertEqual(mime_type, 'testing/default-type') + + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, None) + + mimetypes.add_type('testing/type', '.myext') + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, 'testing/type') + + def test_add_type_with_undotted_extension_not_supported(self): + msg = "Extension 'undotted' must start with '.'" + with self.assertRaisesRegex(ValueError, msg): + mimetypes.add_type("testing/type", "undotted") + with self.assertRaisesRegex(ValueError, msg): + mimetypes.add_type("", "undotted") + + +class MimeTypesClassTestCase(unittest.TestCase): + def setUp(self): + self.db = mimetypes.MimeTypes() + + def test_init_files(self): + guess_file_type = self.db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + db = mimetypes.MimeTypes([filename]) + guess_file_type = db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + def test_init_knownfiles(self): + filename = support.findfile("mime.types2") + mimetypes.knownfiles = [filename, "non-existent"] + self.addCleanup(mimetypes.knownfiles.clear) + + db = mimetypes.MimeTypes() + guess_file_type = db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + def test_case_sensitivity(self): + eq = self.assertEqual + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) + + def test_suffix_map_case_sensitive_preferred(self): + self.db.suffix_map[".TEST-SUFFIX"] = ".tar.gz" + self.db.suffix_map[".test-suffix"] = ".tar.xz" + self.assertEqual( + self.db.guess_file_type("example.TEST-SUFFIX"), + ("application/x-tar", "gzip"), + ) + self.assertEqual( + self.db.guess_file_type("example.test-suffix"), + ("application/x-tar", "xz"), + ) + + def test_added_types_case_sensitive_preferred(self): + self.db.add_type("text/x-test-uppercase-r", ".R") + self.db.add_type("text/x-test-lowercase-r", ".r") + self.assertEqual( + self.db.guess_file_type("example.R"), + ("text/x-test-uppercase-r", None), + ) + self.assertEqual( + self.db.guess_file_type("example.r"), + ("text/x-test-lowercase-r", None), + ) + self.db.add_type("text/x-test-uppercase-non-strict", + ".NON-STRICT-EXT", strict=False) + self.db.add_type("text/x-test-lowercase-non-strict", + ".non-strict-ext", strict=False) + self.assertEqual( + self.db.guess_file_type("example.NON-STRICT-EXT"), + (None, None), + ) + self.assertEqual( + self.db.guess_file_type("example.non-strict-ext"), + (None, None), + ) + self.assertEqual( + self.db.guess_file_type("example.NON-STRICT-EXT", strict=False), + ("text/x-test-uppercase-non-strict", None), + ) + self.assertEqual( + self.db.guess_file_type("example.non-strict-ext", strict=False), + ("text/x-test-lowercase-non-strict", None), + ) + + def test_default_data(self): + eq = self.assertEqual + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) + + def test_data_urls(self): + eq = self.assertEqual + guess_type = self.db.guess_type + eq(guess_type("data:invalidDataWithoutComma"), (None, None)) + eq(guess_type("data:,thisIsTextPlain"), ("text/plain", None)) + eq(guess_type("data:;base64,thisIsTextPlain"), ("text/plain", None)) + eq(guess_type("data:text/x-foo,thisIsTextXFoo"), ("text/x-foo", None)) + + def test_file_parsing(self): + eq = self.assertEqual + sio = io.StringIO("x-application/x-unittest pyunit\n") + self.db.readfp(sio) + eq(self.db.guess_file_type("foo.pyunit"), + ("x-application/x-unittest", None)) + eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") + + def test_non_standard_types(self): + eq = self.assertEqual + # First try strict + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) + # And then non-strict + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_extension('image/jpeg', strict=False), '.jpg') + eq(self.db.guess_extension('image/JPEG', strict=False), '.jpg') + + def test_filename_with_url_delimiters(self): + # bpo-38449: URL delimiters cases should be handled also. + # They would have different mime types if interpreted as URL as + # compared to when interpreted as filename because of the semicolon. + eq = self.assertEqual + gzip_expected = ('application/x-tar', 'gzip') + for name in ( + ';1.tar.gz', + '?1.tar.gz', + '#1.tar.gz', + '#1#.tar.gz', + ';1#.tar.gz', + ';&1=123;?.tar.gz', + '?k1=v1&k2=v2.tar.gz', + ): + for prefix in ('', '/', '\\', + 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', + '//share/server/', '\\\\share\\server\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) + eq(self.db.guess_type(path), gzip_expected) + expected = (None, None) if os.name == 'nt' else gzip_expected + for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) + eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') + result = self.db.guess_type('http://host.html') + msg = 'URL only has a host name, not a file' + self.assertSequenceEqual(result, (None, None), msg) + result = self.db.guess_type('http://example.com/host.html') + msg = 'Should be text/html' + self.assertSequenceEqual(result, ('text/html', None), msg) + result = self.db.guess_type('http://example.com/host.html#x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + result = self.db.guess_type('http://example.com/host.html?q=x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + + def test_guess_all_types(self): + # First try strict. Use a set here for testing the results because if + # test_urllib2 is run before test_mimetypes, global state is modified + # such that the 'all' set will have more items in it. + all = self.db.guess_all_extensions('text/plain', strict=True) + self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) + self.assertEqual(len(set(all)), len(all)) # no duplicates + # And now non-strict + all = self.db.guess_all_extensions('image/jpeg', strict=False) + self.assertEqual(all, ['.jpg', '.jpe', '.jpeg']) + # And now for no hits + all = self.db.guess_all_extensions('image/jpg', strict=True) + self.assertEqual(all, []) + # And now for type existing in both strict and non-strict mappings. + self.db.add_type('test-type', '.strict-ext') + self.db.add_type('test-type', '.non-strict-ext', strict=False) + all = self.db.guess_all_extensions('test-type', strict=False) + self.assertEqual(all, ['.strict-ext', '.non-strict-ext']) + all = self.db.guess_all_extensions('test-type') + self.assertEqual(all, ['.strict-ext']) + # Test that changing the result list does not affect the global state + all.append('.no-such-ext') + all = self.db.guess_all_extensions('test-type') + self.assertNotIn('.no-such-ext', all) + + def test_encoding(self): + filename = support.findfile("mime.types") + mimes = mimetypes.MimeTypes([filename]) + exts = mimes.guess_all_extensions('application/vnd.geocube+xml', + strict=True) + self.assertEqual(exts, ['.g3', '.g\xb3']) + def test_path_like_ob(self): filename = "LICENSE.txt" filepath = os_helper.FakePath(filename) @@ -377,22 +488,6 @@ def test_keywords_args_api(self): self.assertEqual(self.db.guess_all_extensions( type='image/jpeg', strict=True), ['.jpg', '.jpe', '.jpeg']) - def test_added_types_are_used(self): - mimetypes.add_type('testing/default-type', '') - mime_type, _ = mimetypes.guess_type('') - self.assertEqual(mime_type, 'testing/default-type') - - mime_type, _ = mimetypes.guess_type('test.myext') - self.assertEqual(mime_type, None) - - mimetypes.add_type('testing/type', '.myext') - mime_type, _ = mimetypes.guess_type('test.myext') - self.assertEqual(mime_type, 'testing/type') - - def test_add_type_with_undotted_extension_deprecated(self): - with self.assertWarns(DeprecationWarning): - mimetypes.add_type("testing/type", "undotted") - @unittest.skipUnless(sys.platform.startswith("win"), "Windows only") class Win32MimeTypesTestCase(unittest.TestCase): @@ -508,5 +603,59 @@ def test_invocation_error(self): self.assertEqual(result, expected) +@requires_subprocess() +class CommandLineSubprocessTest(unittest.TestCase): + def test_help(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '--help') + self.assertIn(b'mimetypes', stdout) + self.assertIn(b'--extension', stdout) + self.assertIn(b'--lenient', stdout) + + def test_type_lookup(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', 'foo.pdf') + self.assertEqual(stdout.strip(), b'type: application/pdf encoding: None') + self.assertEqual(stderr, b'') + + def test_type_lookup_unknown(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', 'foo.unknownext12345') + self.assertEqual(stdout.strip(), b'error: media type unknown for foo.unknownext12345') + self.assertEqual(stderr, b'') + + def test_extension_flag(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '-e', 'image/jpeg') + self.assertEqual(stdout.strip(), b'.jpg') + self.assertEqual(stderr, b'') + + def test_extension_flag_unknown(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', '-e', 'image/unknowntype12345') + self.assertEqual(stdout.strip(), b'error: unknown type image/unknowntype12345') + self.assertEqual(stderr, b'') + + def test_lenient_flag(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '-e', '--lenient', 'text/xul') + self.assertIn(b'.xul', stdout) + self.assertEqual(stderr, b'') + + def test_multiple_inputs(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', 'foo.pdf', 'foo.png') + self.assertIn(b'type: application/pdf encoding: None', stdout) + self.assertIn(b'type: image/png encoding: None', stdout) + self.assertEqual(stderr, b'') + + def test_multiple_inputs_with_error(self): + rc, stdout, stderr = assert_python_failure( + '-m', 'mimetypes', 'foo.pdf', 'foo.unknownext12345' + ) + self.assertIn(b'type: application/pdf encoding: None', stdout) + self.assertIn(b'error: media type unknown for foo.unknownext12345', stdout) + self.assertEqual(stderr, b'') + + @force_not_colorized + def test_unknown_flag(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', '--unknown-flag', 'foo.pdf') + self.assertEqual(stdout, b'') + self.assertIn(b'error: unrecognized arguments: --unknown-flag', stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 177fe45e8d97490..2e2ac147968dd4a 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -354,6 +354,8 @@ def test_find_end(self): self.assertEqual(m.find(b'one', 1, -1), 8) self.assertEqual(m.find(b'one', 1, -2), -1) self.assertEqual(m.find(bytearray(b'one')), 0) + self.assertEqual(m.find(b'', n + 1), -1) + self.assertEqual(m.rfind(b'', n + 1), -1) for i in range(-n-1, n+1): for j in range(-n-1, n+1): diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index 1c6905bd1ee5864..fef86ce323e54d5 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -4,6 +4,7 @@ import unittest from textwrap import dedent +from test import support from test.support import os_helper, requires_resource from test.support.os_helper import TESTFN, TESTFN_ASCII @@ -67,8 +68,12 @@ def run_in_separated_process(self, code): # Run test in a separated process to avoid stdin conflicts. # See: gh-110147 cmd = [sys.executable, '-c', code] - subprocess.run(cmd, check=True, capture_output=True, - creationflags=subprocess.CREATE_NEW_CONSOLE) + try: + subprocess.run(cmd, check=True, capture_output=True, + creationflags=subprocess.CREATE_NEW_CONSOLE) + except subprocess.CalledProcessError as exc: + support.skip_on_low_desktop_heap_memory_subprocess(exc.returncode) + raise def test_kbhit(self): code = dedent(''' diff --git a/Lib/test/test_multiprocessing_forkserver/__init__.py b/Lib/test/test_multiprocessing_forkserver/__init__.py index 7b1b884ab297b57..c58375e2861c62e 100644 --- a/Lib/test/test_multiprocessing_forkserver/__init__.py +++ b/Lib/test/test_multiprocessing_forkserver/__init__.py @@ -6,7 +6,7 @@ if support.PGO: raise unittest.SkipTest("test is not helpful for PGO") -if sys.platform == "win32": +if sys.platform in ("win32", "cygwin"): raise unittest.SkipTest("forkserver is not available on Windows") if not support.has_fork_support: diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 9480bf8b87baf35..7946550ec0db637 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1419,6 +1419,30 @@ def binary_op_add_extend(): self.assertEqual(c, 2.0) c = b / a self.assertEqual(c, 0.5) + c = a + c += b + self.assertEqual(c, 9.0) + c = b + c += a + self.assertEqual(c, 9.0) + c = a + c -= b + self.assertEqual(c, 3.0) + c = b + c -= a + self.assertEqual(c, -3.0) + c = a + c *= b + self.assertEqual(c, 18.0) + c = b + c *= a + self.assertEqual(c, 18.0) + c = a + c /= b + self.assertEqual(c, 2.0) + c = b + c /= a + self.assertEqual(c, 0.5) binary_op_add_extend() self.assert_specialized(binary_op_add_extend, "BINARY_OP_EXTEND") @@ -1637,6 +1661,37 @@ def send_yield_from(): self.assert_specialized(send_yield_from, "SEND_GEN") self.assert_no_opcode(send_yield_from, "SEND") + @cpython_only + @requires_specialization + def test_send_yield_from_iter(self): + L = list(range(100)) + def send_yield_from(): + yield from L + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + list(send_yield_from()) + + self.assert_specialized(send_yield_from, "SEND_VIRTUAL") + self.assert_no_opcode(send_yield_from, "SEND") + + @cpython_only + @requires_specialization + def test_send_async_for(self): + async def g(): + yield None + + async def send_for(): + async for _ in g(): + break + + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + try: + send_for().send(None) + except StopIteration: + pass + self.assert_specialized(send_for, "SEND_ASYNC_GEN") + self.assert_no_opcode(send_for, "SEND") + @cpython_only @requires_specialization def test_store_attr_slot(self): @@ -2154,6 +2209,17 @@ def load_module_attr_missing(): finally: sys.modules.pop("test_module_with_getattr", None) + @cpython_only + @requires_specialization + def test_specialized_iter_doesnt_skip_send_check(self): + def gen_func(seq): + yield from seq + gen = gen_func(list(range(10))) + for _ in range(3): + gen.send(None) + with self.assertRaises(AttributeError): + gen.send(1) + @cpython_only @requires_specialization diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 7e670e5a139d999..b88388e091312a0 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -1867,7 +1867,9 @@ def test_walk_bad_dir2(self): walk_it = self.walk(self.tmp1_path, follow_symlinks=True) if self.is_fwalk: - self.assertRaises(NotADirectoryError, next, walk_it) + with self.assertRaises(OSError) as cm: + next(walk_it) + self.assertIn(cm.exception.errno, (errno.ENOTDIR, errno.EINVAL)) self.assertRaises(StopIteration, next, walk_it) @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') @@ -2137,6 +2139,94 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mode_with_parent_mode(self): + # Test the parent_mode parameter + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o002): + # Specify mode for both leaf and parent directories + os.makedirs(path, 0o770, parent_mode=0o750) + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + # Leaf directory gets the mode parameter + self.assertEqual(os.stat(path).st_mode & 0o777, 0o770) + # Parent directory gets the parent_mode parameter + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o750) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_deep_hierarchy(self): + # Test parent_mode with deep directory hierarchy + base = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3') + with os_helper.temp_umask(0o002): + os.makedirs(base, 0o755, parent_mode=0o700) + self.assertTrue(os.path.exists(base)) + if os.name != 'nt': + # Check that all parent directories have parent_mode + level1 = os.path.join(os_helper.TESTFN, 'dir1') + level2 = os.path.join(level1, 'dir2') + self.assertEqual(os.stat(level1).st_mode & 0o777, 0o700) + self.assertEqual(os.stat(level2).st_mode & 0o777, 0o700) + # Leaf directory has the regular mode + self.assertEqual(os.stat(base).st_mode & 0o777, 0o755) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_same_as_mode(self): + # Test emulating Python 3.6 behavior by setting parent_mode=mode + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o002): + os.makedirs(path, 0o705, parent_mode=0o705) + self.assertTrue(os.path.exists(path)) + if os.name != 'nt': + # Both directories should have the same mode + self.assertEqual(os.stat(path).st_mode & 0o777, 0o705) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o705) + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_parent_mode_combined_with_umask(self): + # parent_mode, like mode, is combined with the process umask; it does + # not bypass it. + parent = os.path.join(os_helper.TESTFN, 'dir1') + path = os.path.join(parent, 'dir2') + with os_helper.temp_umask(0o022): + os.makedirs(path, 0o777, parent_mode=0o777) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + # 0o777 is masked down to 0o755 by the 0o022 umask, for both + # the leaf (mode) and the parent (parent_mode). + self.assertEqual(os.stat(path).st_mode & 0o777, 0o755) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o755) + @unittest.skipIf( support.is_wasi, "WASI's umask is a stub." @@ -2210,15 +2300,9 @@ def test_win32_mkdir_700(self): ) def tearDown(self): - path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', - 'dir4', 'dir5', 'dir6') - # If the tests failed, the bottom-most directory ('../dir6') - # may not have been created, so we look for the outermost directory - # that exists. - while not os.path.exists(path) and path != os_helper.TESTFN: - path = os.path.dirname(path) - - os.removedirs(path) + # Remove the whole tree regardless of which sub-directories a test + # created and regardless of their permission bits. + os_helper.rmtree(os_helper.TESTFN) @unittest.skipUnless(hasattr(os, "chown"), "requires os.chown()") @@ -2269,6 +2353,8 @@ def test_chown_with_root(self): @requires_non_root_user @unittest.skipUnless(len(all_users) > 1, "test needs and more than one user") + @unittest.skipIf(sys.platform == 'cygwin', + 'chown() can set any uid on Cygwin') def test_chown_without_permission(self): uid_1, uid_2 = all_users[:2] gid = os.stat(os_helper.TESTFN).st_gid @@ -3707,7 +3793,6 @@ async def test_trailers(self): @requires_headers_trailers @requires_32b async def test_headers_overflow_32bits(self): - self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: await self.async_sendfile(self.sockno, self.fileno, 0, 0, headers=[b"x" * 2**16] * 2**15) @@ -3716,7 +3801,6 @@ async def test_headers_overflow_32bits(self): @requires_headers_trailers @requires_32b async def test_trailers_overflow_32bits(self): - self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: await self.async_sendfile(self.sockno, self.fileno, 0, 0, trailers=[b"x" * 2**16] * 2**15) @@ -4051,10 +4135,11 @@ def test_timerfd_non_blocking(self): initial_expiration = 0.1 os.timerfd_settime(fd, initial=initial_expiration, interval=0) - # read() raises OSError with errno is EAGAIN for non-blocking timer. - with self.assertRaises(OSError) as ctx: - self.read_count_signaled(fd) - self.assertEqual(ctx.exception.errno, errno.EAGAIN) + if sys.platform != 'cygwin': + # read() raises OSError with errno is EAGAIN for non-blocking timer. + with self.assertRaises(OSError) as ctx: + self.read_count_signaled(fd) + self.assertEqual(ctx.exception.errno, errno.EAGAIN) # Wait more than 0.1 seconds time.sleep(initial_expiration + 0.1) @@ -4235,12 +4320,19 @@ def test_timerfd_ns_initval(self): # 2nd call next_expiration_ns, interval_ns2 = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) - self.assertEqual(interval_ns2, interval_ns) + CYGWIN = (sys.platform == 'cygwin') + if not CYGWIN: + self.assertEqual(interval_ns2, interval_ns) + else: + self.assertEqual(interval_ns2, 0) self.assertEqual(next_expiration_ns, initial_expiration_ns) # timerfd_gettime next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) - self.assertEqual(interval_ns2, interval_ns) + if not CYGWIN: + self.assertEqual(interval_ns2, interval_ns) + else: + self.assertEqual(interval_ns2, 0) self.assertLessEqual(next_expiration_ns, initial_expiration_ns) self.assertAlmostEqual(next_expiration_ns, initial_expiration_ns, delta=limit_error) diff --git a/Lib/test/test_os/test_posix.py b/Lib/test/test_os/test_posix.py index 0e8495a4eff2ed4..8e83fa21dae6e22 100644 --- a/Lib/test/test_os/test_posix.py +++ b/Lib/test/test_os/test_posix.py @@ -142,8 +142,8 @@ def test_initgroups(self): self.assertRaises(TypeError, posix.initgroups, "foo", 3, object()) # If a non-privileged user invokes it, it should fail with OSError - # EPERM. - if os.getuid() != 0: + # EPERM. On Cygwin, initgroups(name, 13) does not fail. + if os.getuid() != 0 and sys.platform != 'cygwin': try: name = pwd.getpwuid(posix.getuid()).pw_name except KeyError: @@ -597,7 +597,9 @@ def test_sysconf(self): posix.sysconf(1.23) arg_max = posix.sysconf("SC_ARG_MAX") - self.assertGreater(arg_max, 0) + # SC_ARG_MAX is -1 on Cygwin + if sys.platform != 'cygwin': + self.assertGreater(arg_max, 0) self.assertEqual( posix.sysconf(posix.sysconf_names["SC_ARG_MAX"]), arg_max) @@ -899,7 +901,9 @@ def check_stat(uid, gid): self.assertRaises(OSError, chown_func, first_param, 0, -1) check_stat(uid, gid) if hasattr(os, 'getgroups'): - if 0 not in os.getgroups(): + # Also check the effective gid, which the kernel + # accepts for chown even if not in getgroups(). + if 0 not in os.getgroups() and os.getegid() != 0: self.assertRaises(OSError, chown_func, first_param, -1, 0) check_stat(uid, gid) # test illegal types @@ -1602,6 +1606,34 @@ def test_pidfd_open(self): self.assertEqual(cm.exception.errno, errno.EINVAL) os.close(os.pidfd_open(os.getpid(), 0)) + @unittest.skipUnless(hasattr(os, "pidfd_getfd"), "pidfd_getfd unavailable") + def test_pidfd_getfd(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + pidfd = os.pidfd_open(os.getpid(), 0) + self.addCleanup(os.close, pidfd) + try: + dupfd = os.pidfd_getfd(pidfd, fd) + except OSError as exc: + if exc.errno == errno.ENOSYS: + self.skipTest("system does not support pidfd_getfd") + if isinstance(exc, PermissionError): + self.skipTest(f"pidfd_getfd syscall blocked: {exc!r}") + raise + self.addCleanup(os.close, dupfd) + + self.assertFalse(os.get_inheritable(dupfd)) # PEP 446 + self.assertEqual(os.fstat(fd), os.fstat(dupfd)) + + with self.assertRaises(OSError) as cm: + os.pidfd_getfd(-1, 0) + self.assertEqual(cm.exception.errno, errno.EBADF) + + with self.assertRaises(OSError) as cm: + bad_fd = os_helper.make_bad_fd() + os.pidfd_getfd(pidfd, bad_fd) + self.assertEqual(cm.exception.errno, errno.EBADF) + @os_helper.skip_unless_hardlink @os_helper.skip_unless_symlink def test_link_follow_symlinks(self): @@ -1943,6 +1975,14 @@ def test_no_such_executable(self): # directories in the $PATH that are not accessible. except (FileNotFoundError, PermissionError) as exc: self.assertEqual(exc.filename, no_such_executable) + + # On Cygwin, os.posix_spawn() creates a child process even if the + # executable doesn't exist. We have to reap this process. + if sys.platform == 'cygwin': + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + pid, status = os.waitpid(-1, os.WNOHANG) + if pid != 0: + break else: pid2, status = os.waitpid(pid, 0) self.assertEqual(pid2, pid) diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 19f4506c109c147..aff66c8efedbbc0 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -81,7 +81,7 @@ def test_is_notimplemented(self): class LazyImportTest(unittest.TestCase): @cpython_only def test_lazy_import(self): - import_helper.ensure_lazy_imports("pathlib", {"shutil"}) + import_helper.ensure_lazy_imports("pathlib", {"glob", "shutil"}) # @@ -2492,6 +2492,116 @@ def my_mkdir(path, mode=0o777): self.assertNotIn(str(p12), concurrently_created) self.assertTrue(p.exists()) + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parents_umask(self): + # Test that parent directories respect umask when parent_mode is not set + p = self.cls(self.base, 'umasktest', 'child') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o002): + p.mkdir(0o755, parents=True) + self.assertTrue(p.exists()) + # Leaf directory gets the specified mode + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + # Parent directory respects umask (0o777 & ~0o002 = 0o775) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o775) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_with_parent_mode(self): + # Test the parent_mode parameter + p = self.cls(self.base, 'newdirPM', 'subdirPM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + # Specify different modes for parent and leaf directories + p.mkdir(0o755, parents=True, parent_mode=0o750) + self.assertTrue(p.exists()) + self.assertTrue(p.is_dir()) + # Leaf directory gets the mode parameter + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + # Parent directory gets the parent_mode parameter + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o750) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_deep_hierarchy(self): + # Test parent_mode with deep directory hierarchy + p = self.cls(self.base, 'level1PM', 'level2PM', 'level3PM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o755, parents=True, parent_mode=0o700) + self.assertTrue(p.exists()) + # Check that all parent directories have parent_mode + level1 = self.cls(self.base, 'level1PM') + level2 = level1 / 'level2PM' + self.assertEqual(level1.stat().st_mode & 0o777, 0o700) + self.assertEqual(level2.stat().st_mode & 0o777, 0o700) + # Leaf directory has the regular mode + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_combined_with_umask(self): + # parent_mode, like mode, is combined with the process umask; it does + # not bypass it. + p = self.cls(self.base, 'umaskPM', 'child') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o777, parents=True, parent_mode=0o777) + self.assertTrue(p.exists()) + # 0o777 is masked down to 0o755 by the 0o022 umask, for both + # the leaf (mode) and the parent (parent_mode). + self.assertEqual(p.stat().st_mode & 0o777, 0o755) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o755) + + @unittest.skipIf( + is_emscripten or is_wasi, + "umask is not implemented on Emscripten/WASI." + ) + @unittest.skipIf( + sys.platform == "android", + "Android filesystem may not honor requested permissions." + ) + def test_mkdir_parent_mode_same_as_mode(self): + # Test setting parent_mode same as mode + p = self.cls(self.base, 'samedirPM', 'subdirPM') + self.assertFalse(p.exists()) + if os.name != 'nt': + with os_helper.temp_umask(0o022): + p.mkdir(0o705, parents=True, parent_mode=0o705) + self.assertTrue(p.exists()) + # Both directories should have the same mode + self.assertEqual(p.stat().st_mode & 0o777, 0o705) + self.assertEqual(p.parent.stat().st_mode & 0o777, 0o705) + @needs_symlinks def test_symlink_to(self): P = self.cls(self.base) @@ -2760,6 +2870,7 @@ def test_is_socket_true(self): if (isinstance(e, PermissionError) or "AF_UNIX path too long" in str(e)): self.skipTest("cannot bind Unix socket: " + str(e)) + raise self.assertTrue(P.is_socket()) self.assertFalse(P.is_fifo()) self.assertFalse(P.is_file()) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 29cce4ee6d271ff..e3aaea84ea7ce84 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2852,6 +2852,15 @@ def test_patma_266(self): self.assertEqual(x, 0) self.assertEqual(y, 1) + def test_patma_frozendict_class_self(self): + x = frozendict() + match x: + case frozendict(z): + y = 0 + self.assertEqual(x, frozendict()) + self.assertEqual(y, 0) + self.assertIs(z, x) + def test_patma_runtime_checkable_protocol(self): # Runtime-checkable protocol from typing import Protocol, runtime_checkable diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index db90019975521ea..410f1436ed4d20a 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -4718,6 +4718,27 @@ def bar(): ])) self.assertIn('break in bar', stdout) + def test_end_of_options_separator(self): + # gh-148615: Test parsing when '--' separator is used + script = "import sys; print(f'ARGS: {sys.argv[1:]}')" + with open(os_helper.TESTFN, 'w', encoding='utf-8') as f: + f.write(script) + stdout, _ = self._run_pdb(['--', os_helper.TESTFN, '-foo'], 'c\nq') + self.assertIn("ARGS: ['-foo']", stdout) + stdout, _ = self._run_pdb(['-c', 'continue', '--', os_helper.TESTFN, '-c', 'foo'], 'q') + self.assertIn("ARGS: ['-c', 'foo']", stdout) + stdout, stderr = self._run_pdb(['--'], 'q', expected_returncode=2) + self.assertIn("missing script or module to run", stderr) + stdout, stderr = self._run_pdb(['-x', '--', os_helper.TESTFN], 'q', expected_returncode=2) + self.assertIn("unrecognized arguments: -x", stderr) + stdout, _ = self._run_pdb([os_helper.TESTFN, '--', 'arg'], 'c\nq') + self.assertIn("ARGS: ['--', 'arg']", stdout) + with os_helper.temp_cwd(): + with open('mymod.py', 'w', encoding='utf-8') as f: + f.write(script) + stdout, _ = self._run_pdb(['-m', 'mymod', '--', 'arg'], 'c\nq') + self.assertIn("ARGS: ['--', 'arg']", stdout) + @unittest.skipIf(SKIP_CORO_TESTS, "Coroutine tests are skipped") def test_async_break(self): script = """ @@ -4981,6 +5002,29 @@ def test_stack_entry(self): p.set_trace(commands=['w', 'c']) self.assertIn("\x1b", output.getvalue()) + @unittest.skipIf(not pdb._pyrepl_available(), "pyrepl is not available") + def test_gen_colors(self): + p = pdb.Pdb() + gen_colors = p.pyrepl_input.gen_colors + + test_cases = [ + ("longlist", [((0, 7), "soft_keyword")]), + ("!longlist", [((0, 0), "op")]), + ("list", [((0, 3), "soft_keyword")]), + ("list(", [((0, 3), "builtin"), ((4, 4), "op")]), + ("a = 1", [ + ((0, 0), "soft_keyword"), + ((2, 2), "op"), + ((4, 4), "number"), + ]) + ] + + for buffer, expected in test_cases: + for color_span, ((start, end), tag) in zip(gen_colors(buffer), expected, strict=True): + self.assertEqual(color_span.span.start, start) + self.assertEqual(color_span.span.end, end) + self.assertEqual(color_span.tag, tag) + @support.force_not_colorized_test_class @support.requires_subprocess() diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index d44e61f25f7f552..28748009f731bc1 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1127,8 +1127,8 @@ def test_import_from_doesnt_clobber_load_fast_borrow(self): def f(self): if x: pass self.x - from shutil import ExecError - print(ExecError) + from heapq import heapify_max + print(heapify_max) self.assertInBytecode(f, "LOAD_FAST_BORROW", "self") class DirectCfgOptimizerTests(CfgOptimizationTestCase): @@ -1307,10 +1307,10 @@ def test_build_empty_tuple(self): ('RETURN_VALUE', None, 0), ] after = [ - ('LOAD_CONST', 0, 0), + ('LOAD_COMMON_CONSTANT', opcode._common_constants.index(()), 0), ('RETURN_VALUE', None, 0), ] - self.cfg_optimization_test(before, after, consts=[], expected_consts=[()]) + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) def test_fold_tuple_of_constants(self): before = [ @@ -1365,10 +1365,10 @@ def test_fold_constant_intrinsic_list_to_tuple(self): ('RETURN_VALUE', None, 0) ] after = [ - ('LOAD_CONST', 0, 0), + ('LOAD_COMMON_CONSTANT', opcode._common_constants.index(()), 0), ('RETURN_VALUE', None, 0) ] - self.cfg_optimization_test(before, after, consts=[], expected_consts=[()]) + self.cfg_optimization_test(before, after, consts=[], expected_consts=[]) # multiple BUILD_LIST 0: ([], 1, [], 2) same = [ diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 597e65993520491..425c76dd01ed7c2 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -34,6 +34,21 @@ def supports_trampoline_profiling(): raise unittest.SkipTest("perf trampoline profiling not supported") +def _perf_env(**env_vars): + env = os.environ.copy() + # Keep perf's output stable regardless of the builder's perf config. + env.update( + { + "DEBUGINFOD_URLS": "", + "PERF_CONFIG": os.devnull, + } + ) + if env_vars: + env.update(env_vars) + env["PYTHON_JIT"] = "0" + return env + + class TestPerfTrampoline(unittest.TestCase): def setUp(self): super().setUp() @@ -63,13 +78,12 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -132,13 +146,12 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -198,13 +211,12 @@ def test_trampoline_works_after_fork_with_many_code_objects(self): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -242,13 +254,12 @@ def baz(): code = set_eval_hook + code with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -345,9 +356,12 @@ def perf_command_works(): "-c", 'print("hello")', ) - env = {**os.environ, "PYTHON_JIT": "0"} stdout = subprocess.check_output( - cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env + cmd, + cwd=script_dir, + text=True, + stderr=subprocess.STDOUT, + env=_perf_env(), ) except (subprocess.SubprocessError, OSError): return False @@ -359,43 +373,49 @@ def perf_command_works(): def run_perf(cwd, *args, use_jit=False, **env_vars): - env = os.environ.copy() - if env_vars: - env.update(env_vars) - env["PYTHON_JIT"] = "0" + env = _perf_env(**env_vars) output_file = cwd + "/perf_output.perf" - if not use_jit: - base_cmd = ( - "perf", - "record", - "--no-buildid", - "--no-buildid-cache", - "-g", - "--call-graph=fp", - "-o", output_file, - "--" - ) + base_cmd = [ + "perf", + "record", + "--no-buildid", + "--no-buildid-cache", + "-g", + "--call-graph=dwarf,65528" if use_jit else "--call-graph=fp", + ] + if use_jit: + perf_commands = [] + # Some builders have low perf_event_mlock_kb limits. + mmap_sizes = ("4M", "2M", "1M", "512K", "256K", "128K", None) + for mmap_size in mmap_sizes: + command = base_cmd.copy() + if mmap_size is not None: + command += ["-F99", "-k1", "-m", mmap_size] + else: + command += ["-F99", "-k1"] + command += ["-o", output_file, "--"] + perf_commands.append(command) else: - base_cmd = ( - "perf", - "record", - "--no-buildid", - "--no-buildid-cache", - "-g", - "--call-graph=dwarf,65528", - "-F99", - "-k1", - "-o", - output_file, - "--", + perf_commands = [base_cmd + ["-o", output_file, "--"]] + + mmap_pages_error = "try again with a smaller value of -m/--mmap_pages" + for index, base_cmd in enumerate(perf_commands): + proc = subprocess.run( + base_cmd + list(args), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + text=True, ) - proc = subprocess.run( - base_cmd + args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - text=True, - ) + if ( + proc.returncode + and use_jit + and index != len(perf_commands) - 1 + and mmap_pages_error in proc.stderr + ): + continue + break + if proc.returncode: print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") @@ -425,16 +445,34 @@ def run_perf(cwd, *args, use_jit=False, **env_vars): class TestPerfProfilerMixin: - def run_perf(self, script_dir, perf_mode, script): + PERF_CAPTURE_ATTEMPTS = 3 + + def run_perf(self, script_dir, script, activate_trampoline=True): raise NotImplementedError() + def run_perf_with_retries( + self, script_dir, script, expected_symbols=(), activate_trampoline=True + ): + stdout = stderr = "" + for _ in range(self.PERF_CAPTURE_ATTEMPTS): + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=activate_trampoline + ) + if activate_trampoline and any( + symbol not in stdout for symbol in expected_symbols + ): + continue + break + return stdout, stderr + def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: code = """if 1: + from itertools import repeat + def foo(n): - x = 0 - for i in range(n): - x += i + for _ in repeat(None, n): + pass def bar(n): foo(n) @@ -442,23 +480,29 @@ def bar(n): def baz(n): bar(n) - baz(10000000) + baz(40000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = self.run_perf(script_dir, script) - self.assertEqual(stderr, "") + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] + stdout, _ = self.run_perf_with_retries( + script_dir, script, expected_symbols + ) - self.assertIn(f"py::foo:{script}", stdout) - self.assertIn(f"py::bar:{script}", stdout) - self.assertIn(f"py::baz:{script}", stdout) + for expected_symbol in expected_symbols: + self.assertIn(expected_symbol, stdout) def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: + from itertools import repeat + def foo(n): - x = 0 - for i in range(n): - x += i + for _ in repeat(None, n): + pass def bar(n): foo(n) @@ -466,13 +510,12 @@ def bar(n): def baz(n): bar(n) - baz(10000000) + baz(40000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = self.run_perf( + stdout, _ = self.run_perf_with_retries( script_dir, script, activate_trampoline=False ) - self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) @@ -542,13 +585,12 @@ def compile_trampolines_for_all_functions(): with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], universal_newlines=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 9ee97b922ad48e1..3503eaefaf8e9e7 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -534,7 +534,20 @@ def test_ios_ver(self): def test_libc_ver(self): if support.is_emscripten: - assert platform.libc_ver() == ("emscripten", "4.0.12") + import tomllib + from pathlib import Path + + # Get expected emscripten version from emscripten config + config_path = ( + Path(__file__).parents[2] / "Platforms/emscripten/config.toml" + ) + with open(config_path, "rb") as fp: + emscripten_version = tomllib.load(fp)["emscripten-version"] + + self.assertEqual( + platform.libc_ver(), ("emscripten", emscripten_version) + ) + return # check that libc_ver(executable) doesn't raise an exception if os.path.isdir(sys.executable) and \ diff --git a/Lib/test/test_profiling/test_heatmap.py b/Lib/test/test_profiling/test_heatmap.py index b2acb1cf577341d..ee27fdd3fa3053c 100644 --- a/Lib/test/test_profiling/test_heatmap.py +++ b/Lib/test/test_profiling/test_heatmap.py @@ -345,6 +345,21 @@ def test_process_frames_tracks_edge_samples(self): # Check that edge count is tracked self.assertGreater(len(collector.edge_samples), 0) + def test_process_frames_weight_applies_to_identical_samples(self): + collector = HeatmapCollector(sample_interval_usec=100) + + frames = [ + ('callee.py', (5, 5, -1, -1), 'callee', None), + ('caller.py', (10, 10, -1, -1), 'caller', None), + ] + + collector.process_frames(frames, thread_id=1, weight=5) + + edge_key = (('caller.py', 10), ('callee.py', 5)) + self.assertEqual(collector.edge_samples[edge_key], 5) + self.assertEqual(collector.line_samples[('callee.py', 5)], 5) + self.assertEqual(collector.line_samples[('caller.py', 10)], 5) + def test_process_frames_handles_empty_frames(self): """Test that process_frames handles empty frame list.""" collector = HeatmapCollector(sample_interval_usec=100) diff --git a/Lib/test/test_profiling/test_sampling_profiler/helpers.py b/Lib/test/test_profiling/test_sampling_profiler/helpers.py index 0e32d8dd9eabef1..b07776d415bb297 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/helpers.py +++ b/Lib/test/test_profiling/test_sampling_profiler/helpers.py @@ -174,3 +174,29 @@ def close_and_unlink(file): """Close a file and unlink it from the filesystem.""" file.close() unlink(file.name) + + +def jsonl_tables(records): + """Extract the canonical sections of a parsed JSONL profile. + + Returns ``(meta, str_defs, frame_defs, agg, end)`` where ``str_defs`` is a + ``{str_id: value}`` dict, ``frame_defs`` is a flat list of all frame + definitions across chunks, and ``agg`` is the first agg record (sufficient + for tests that only emit one chunk). + """ + meta = next(record for record in records if record["type"] == "meta") + end = next(record for record in records if record["type"] == "end") + agg = next(record for record in records if record["type"] == "agg") + str_defs = { + item["str_id"]: item["value"] + for record in records + if record["type"] == "string_table" + for item in record["strings"] + } + frame_defs = [ + item + for record in records + if record["type"] == "frame_table" + for item in record["frames"] + ] + return meta, str_defs, frame_defs, agg, end diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py index 29f83c843561cd0..5efc60a92111754 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_binary_format.py @@ -1,7 +1,10 @@ """Tests for binary format round-trip functionality.""" +import json import os +import pathlib import random +import struct import tempfile import unittest from collections import defaultdict @@ -21,7 +24,7 @@ THREAD_STATUS_MAIN_THREAD, ) from profiling.sampling.binary_collector import BinaryCollector - from profiling.sampling.binary_reader import BinaryReader + from profiling.sampling.binary_reader import BinaryReader, convert_binary_to_format from profiling.sampling.gecko_collector import GeckoCollector ZSTD_AVAILABLE = _remote_debugging.zstd_available() @@ -30,6 +33,8 @@ "Test only runs when _remote_debugging is available" ) +from .helpers import jsonl_tables + def make_frame(filename, lineno, funcname, end_lineno=None, column=None, end_column=None, opcode=None): @@ -148,6 +153,11 @@ def tearDown(self): def create_binary_file(self, samples, interval=1000, compression="none"): """Create a test binary file and track it for cleanup.""" + filename, _ = self.write_binary_file(samples, interval, compression) + return filename + + def write_binary_file(self, samples, interval=1000, compression="none"): + """Like create_binary_file but also returns the writer collector.""" with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: filename = f.name self.temp_files.append(filename) @@ -158,7 +168,7 @@ def create_binary_file(self, samples, interval=1000, compression="none"): for sample in samples: collector.collect(sample) collector.export(None) - return filename + return filename, collector def roundtrip(self, samples, interval=1000, compression="none"): """Write samples to binary and read back.""" @@ -805,6 +815,232 @@ def test_invalid_file_path(self): with BinaryReader("/nonexistent/path/file.bin") as reader: reader.replay_samples(RawCollector()) + def test_path_arguments_round_trip(self): + """Reader and writer accept str, bytes or os.PathLike.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + for path_arg in (filename, os.fsencode(filename), pathlib.Path(filename)): + with self.subTest(path_type=type(path_arg).__name__): + writer = _remote_debugging.BinaryWriter(path_arg, 1000, 0) + writer.finalize() + reader = _remote_debugging.BinaryReader(path_arg) + info = reader.get_info() + reader.close() + self.assertEqual(info["sample_count"], 0) + + def test_rejects_non_pathlike(self): + """Reader and writer raise TypeError on non-path-like filenames.""" + with self.assertRaises(TypeError): + _remote_debugging.BinaryWriter(123, 1000, 0) + with self.assertRaises(TypeError): + _remote_debugging.BinaryReader(123) + + def test_invalid_path_error_preserves_pathlib(self): + """Missing path: OSError carries the original path object, not a string.""" + missing = pathlib.Path("/i/do/not/exist") + with self.assertRaises(FileNotFoundError) as cm: + _remote_debugging.BinaryReader(missing) + self.assertEqual(os.fspath(cm.exception.filename), os.fspath(missing)) + + def test_writer_handles_empty_stack_first_sample(self): + """BinaryWriter.write_sample tolerates an empty stack on a fresh thread. + + Regression test for the C-level RLE bug in process_thread_sample: a + freshly-created ThreadEntry has prev_stack_depth == 0, so an empty + curr_stack compares as STACK_REPEAT against the zero-initialized + previous stack. Before the fix, this fell through the + `&& !is_new_thread` guard into write_sample_with_encoding, which had + no handler for STACK_REPEAT and raised + RuntimeError("Invalid stack encoding type"). Goes through + BinaryWriter.write_sample directly so the test cannot be masked by + any Python-level filtering. + """ + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + writer = _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) + empty_sample = [ + make_interpreter( + 0, [make_thread(99, [], status=THREAD_STATUS_UNKNOWN)] + ) + ] + # First sample for a fresh thread has empty frame_info — the exact + # scenario that exposes the bug. + writer.write_sample(empty_sample, 1000) + writer.write_sample(empty_sample, 2000) + # Mix in a real sample to exercise the transition out of the + # empty-stack RLE buffer. + real_sample = [ + make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + writer.write_sample(real_sample, 3000) + writer.finalize() + + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + count = reader.replay_samples(reader_collector) + # Empty-stack samples are recorded as STACK_REPEAT records with + # depth-0 stacks; the file must replay all three samples. + self.assertEqual(count, 3) + + def test_writer_handles_mixed_empty_and_real_first_sample(self): + """First sample with one empty + one real thread roundtrips through C.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + writer = _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) + sample = [ + make_interpreter( + 0, + [ + make_thread(1, [make_frame("a.py", 1, "f")]), + make_thread(99, [], status=THREAD_STATUS_UNKNOWN), + ], + ) + ] + # Two samples so RLE state is exercised. + writer.write_sample(sample, 1000) + writer.write_sample(sample, 2000) + writer.finalize() + + # Replay must succeed without raising RuntimeError, and the real + # thread's frames must round-trip. + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + reader.replay_samples(reader_collector) + self.assertIn((0, 1), reader_collector.by_thread) + self.assertEqual(len(reader_collector.by_thread[(0, 1)]), 2) + + def test_writer_total_samples_after_finalize_matches_reader(self): + """BinaryWriter.total_samples after finalize() matches the reader's count.""" + # Five IDENTICAL samples force every sample beyond the first into the + # per-thread RLE buffer. Regression for the cached_total_samples + # ordering bug: capturing the cache BEFORE binary_writer_finalize() + # missed the buffered samples that flush_pending_rle() counts. Keep + # the samples identical to preserve coverage. See gh-149342. + samples = [ + [make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])])] + ] * 5 + filename, writer_collector = self.write_binary_file(samples) + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + replayed = reader.replay_samples(reader_collector) + self.assertEqual(writer_collector.total_samples, len(samples)) + self.assertEqual(writer_collector.total_samples, replayed) + + def test_writer_total_samples_after_context_manager_matches_reader(self): + """total_samples after `with BinaryWriter(...)` matches the reader's count. + + Regression for the asymmetry between finalize() and __exit__ in + module.c: __exit__ also calls binary_writer_finalize and must + preserve cached_total_samples like finalize() does, otherwise the + getter returns 0 once self->writer is NULL. + """ + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + sample = [ + make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + with _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) as w: + for i in range(5): + w.write_sample(sample, i * 1000) + self.assertEqual(w.total_samples, 5) + + reader_collector = RawCollector() + with BinaryReader(filename) as reader: + self.assertEqual(reader.replay_samples(reader_collector), 5) + + def test_writer_total_samples_after_close_returns_zero(self): + """close() discards data; total_samples reflects no cached count.""" + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as f: + filename = f.name + self.temp_files.append(filename) + + w = _remote_debugging.BinaryWriter(filename, 1000, 0, compression=0) + sample = [ + make_interpreter(0, [make_thread(1, [make_frame("a.py", 1, "f")])]) + ] + for i in range(5): + w.write_sample(sample, i * 1000) + w.close() + self.assertEqual(w.total_samples, 0) + + +class TestBinaryFormatValidation(BinaryFormatTestBase): + """Tests for malformed binary files.""" + + HDR_OFF_SAMPLES = 28 + HDR_OFF_THREADS = 32 + HDR_OFF_STR_TABLE = 36 + HDR_OFF_FRAME_TABLE = 44 + FILE_HEADER_PLACEHOLDER_SIZE = 64 + + def test_replay_rejects_more_threads_than_declared(self): + """Replay rejects files with more unique threads than the header declares.""" + threads = [ + make_thread(1, [make_frame("t1.py", 10, "t1")]), + make_thread(2, [make_frame("t2.py", 20, "t2")]), + ] + samples = [[make_interpreter(0, threads)]] + filename = self.create_binary_file(samples, compression="none") + + with open(filename, "r+b") as raw: + raw.seek(self.HDR_OFF_THREADS) + raw.write(struct.pack("=I", 1)) + + with BinaryReader(filename) as reader: + self.assertEqual(reader.get_info()["thread_count"], 1) + with self.assertRaises(ValueError) as cm: + reader.replay_samples(RawCollector()) + self.assertEqual( + str(cm.exception), + "Invalid thread count: sample data contains more unique " + "threads than declared in header (declared 1, found at least 2)", + ) + + def test_replay_rejects_sample_count_mismatch(self): + """Replay rejects files whose decoded samples disagree with the header.""" + samples = [[make_interpreter(0, [ + make_thread(1, [make_frame("sample.py", 10, "sample")]) + ])]] + filename = self.create_binary_file(samples, compression="none") + + with open(filename, "r+b") as raw: + raw.seek(self.HDR_OFF_SAMPLES) + raw.write(struct.pack("=I", 2)) + + with BinaryReader(filename) as reader: + self.assertEqual(reader.get_info()["sample_count"], 2) + with self.assertRaises(ValueError) as cm: + reader.replay_samples(RawCollector()) + self.assertEqual( + str(cm.exception), + "Sample count mismatch: header declares 2 samples " + "but replay decoded 1", + ) + + def test_replay_rejects_trailing_partial_sample_header(self): + """Replay rejects partial sample bytes instead of silently stopping.""" + filename = self.create_binary_file([], compression="none") + sample_data_end = self.FILE_HEADER_PLACEHOLDER_SIZE + 1 + + with open(filename, "r+b") as raw: + raw.seek(self.HDR_OFF_STR_TABLE) + raw.write(struct.pack("=Q", sample_data_end)) + raw.seek(self.HDR_OFF_FRAME_TABLE) + raw.write(struct.pack("=Q", sample_data_end)) + + with BinaryReader(filename) as reader: + with self.assertRaises(ValueError) as cm: + reader.replay_samples(RawCollector()) + self.assertEqual(str(cm.exception), "Truncated sample data: 1 trailing bytes") + class TestBinaryEncodings(BinaryFormatTestBase): """Tests specifically targeting different stack encodings.""" @@ -1211,5 +1447,70 @@ def test_timestamp_preservation_with_rle(self): self.assertEqual(ts_collector.all_timestamps, expected_timestamps) +class TestBinaryReplayToJsonl(BinaryFormatTestBase): + """Tests for binary -> JSONL replay via convert_binary_to_format.""" + + def _replay_to_jsonl(self, samples, interval=1000): + bin_path = self.create_binary_file(samples, interval=interval) + with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) as f: + jsonl_path = f.name + self.temp_files.append(jsonl_path) + + convert_binary_to_format(bin_path, jsonl_path, "jsonl") + + with open(jsonl_path, "r", encoding="utf-8") as f: + return [json.loads(line) for line in f] + + def test_binary_replay_to_jsonl_basic(self): + """Replay a small .bin to JSONL: meta/end shape, samples_total, run_id.""" + frame = make_frame("hot.py", 99, "hot_func") + samples = [ + [make_interpreter(0, [make_thread(1, [frame])])] + for _ in range(5) + ] + records = self._replay_to_jsonl(samples, interval=2000) + meta, _, frame_defs, _, end = jsonl_tables(records) + + self.assertEqual(meta["sample_interval_usec"], 2000) + self.assertEqual(end["samples_total"], 5) + + run_ids = {r["run_id"] for r in records} + self.assertEqual(len(run_ids), 1) + self.assertRegex(next(iter(run_ids)), r"^[0-9a-f]{32}$") + + self.assertEqual(len(frame_defs), 1) + self.assertEqual(frame_defs[0]["line"], 99) + + def test_binary_replay_to_jsonl_rle_weight_propagation(self): + """RLE-batched identical samples land as a single agg entry with the right total.""" + frame = make_frame("rle.py", 42, "rle_func") + samples = [ + [make_interpreter(0, [make_thread(1, [frame])])] + for _ in range(50) + ] + records = self._replay_to_jsonl(samples) + _, _, _, agg, end = jsonl_tables(records) + + self.assertEqual(end["samples_total"], 50) + self.assertEqual(agg["entries"], [ + {"frame_id": 0, "self": 50, "cumulative": 50}, + ]) + + def test_binary_replay_to_jsonl_omits_unavailable_columns(self): + """Columns the binary recorder did not capture are omitted, not 0.""" + # make_frame defaults column/end_column to 0; pass column=-1 / end_column=-1 + # so the binary side records LOCATION_NOT_AVAILABLE. + frame = make_frame("nocol.py", 7, "no_col", column=-1, end_column=-1) + samples = [[make_interpreter(0, [make_thread(1, [frame])])]] + records = self._replay_to_jsonl(samples) + _, _, frame_defs, _, _ = jsonl_tables(records) + + self.assertEqual(len(frame_defs), 1) + fd = frame_defs[0] + self.assertEqual(fd["line"], 7) + self.assertNotIn("col", fd) + self.assertNotIn("end_col", fd) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py b/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py index 102eb51b556cc77..1f4b6da32810561 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_blocking.py @@ -39,8 +39,9 @@ class TestBlockingModeStackAccuracy(unittest.TestCase): @classmethod def setUpClass(cls): # Test script that uses a generator consumed in a loop. - # When consume_generator is on the arithmetic lines (temp1, temp2, etc.), - # fibonacci_generator should NOT be in the stack at all. + # When consume_generator is the executing leaf frame on the arithmetic + # lines (temp1, temp2, etc.), fibonacci_generator should NOT be in the + # stack at all. # Line numbers are important here - see ARITHMETIC_LINES below. cls.generator_script = textwrap.dedent(''' def fibonacci_generator(n): @@ -65,29 +66,32 @@ def main(): main() ''') # Line numbers of the arithmetic operations in consume_generator. - # These are the lines where fibonacci_generator should NOT be in the stack. - # The socket injection code adds 7 lines before our script. - # temp1 = value + 1 -> line 17 - # temp2 = value * 2 -> line 18 - # temp3 = value - 1 -> line 19 - # result = ... -> line 20 - cls.ARITHMETIC_LINES = {17, 18, 19, 20} + # These are the lines where fibonacci_generator should NOT be in the + # stack when consume_generator is the executing leaf frame. They account + # for the socket prelude added by test_subprocess(). + # temp1 = value + 1 -> line 16 + # temp2 = value * 2 -> line 17 + # temp3 = value - 1 -> line 18 + # result = ... -> line 19 + cls.ARITHMETIC_LINES = {16, 17, 18, 19} def test_generator_not_under_consumer_arithmetic(self): """Test that fibonacci_generator doesn't appear when consume_generator does arithmetic. - When consume_generator is executing arithmetic lines (temp1, temp2, etc.), - fibonacci_generator should NOT be anywhere in the stack - it's not being - called at that point. + When consume_generator is the leaf frame on arithmetic lines (temp1, + temp2, etc.), fibonacci_generator should NOT be anywhere in the stack - + it's not being called at that point. Non-leaf frame line numbers are + caller/resume metadata, not proof that the frame is executing. Valid stacks: - - consume_generator at 'for value in gen:' line WITH fibonacci_generator - at the top (generator is yielding) + - fibonacci_generator at the top (generator is executing), with + consume_generator below it - consume_generator at arithmetic lines WITHOUT fibonacci_generator (we're just doing math, not calling the generator) Invalid stacks (indicate torn/inconsistent reads): - - consume_generator at arithmetic lines WITH fibonacci_generator + - consume_generator leaf frame at arithmetic lines WITH + fibonacci_generator anywhere in the stack Note: call_tree is ordered from bottom (index 0) to top (index -1). @@ -110,6 +114,8 @@ def test_generator_not_under_consumer_arithmetic(self): total_samples = 0 invalid_stacks = 0 arithmetic_samples = 0 + generator_samples = 0 + generator_not_leaf_samples = 0 for (call_tree, _thread_id), count in collector.stack_counter.items(): total_samples += count @@ -117,15 +123,21 @@ def test_generator_not_under_consumer_arithmetic(self): if not call_tree: continue - # Find consume_generator in the stack and check its line number - for i, (filename, lineno, funcname) in enumerate(call_tree): - if funcname == "consume_generator" and lineno in self.ARITHMETIC_LINES: - arithmetic_samples += count - # Check if fibonacci_generator appears anywhere in this stack - func_names = [frame[2] for frame in call_tree] - if "fibonacci_generator" in func_names: - invalid_stacks += count - break + # Non-leaf frame line numbers can point at resume locations while + # a callee is the executing leaf frame. + _, lineno, funcname = call_tree[-1] + func_names = [frame[2] for frame in call_tree] + + if "fibonacci_generator" in func_names: + generator_samples += count + if funcname != "fibonacci_generator": + generator_not_leaf_samples += count + + if funcname == "consume_generator" and lineno in self.ARITHMETIC_LINES: + arithmetic_samples += count + # Check if fibonacci_generator appears anywhere in this stack. + if "fibonacci_generator" in func_names: + invalid_stacks += count self.assertGreater(total_samples, 10, f"Expected at least 10 samples, got {total_samples}") @@ -134,8 +146,15 @@ def test_generator_not_under_consumer_arithmetic(self): self.assertGreater(arithmetic_samples, 0, f"Expected some samples on arithmetic lines, got {arithmetic_samples}") + self.assertGreater(generator_samples, 0, + f"Expected some samples in fibonacci_generator, got {generator_samples}") + + self.assertEqual(generator_not_leaf_samples, 0, + f"Found {generator_not_leaf_samples}/{generator_samples} stacks where " + f"fibonacci_generator appears but is not the leaf frame.") + self.assertEqual(invalid_stacks, 0, f"Found {invalid_stacks}/{arithmetic_samples} invalid stacks where " f"fibonacci_generator appears in the stack when consume_generator " - f"is on an arithmetic line. This indicates torn/inconsistent stack " - f"traces are being captured.") + f"is the leaf frame on an arithmetic line. This indicates " + f"torn/inconsistent stack traces are being captured.") diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_children.py b/Lib/test/test_profiling/test_sampling_profiler/test_children.py index bb49faa890f3481..e64d917eedde56b 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_children.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_children.py @@ -109,6 +109,39 @@ def _wait_for_process_ready(proc, timeout): return proc.poll() is None +@unittest.skipIf( + _build_child_profiler_args is None, + "profiling.sampling.cli unavailable", +) +class TestChildProfilerArgBuilder(unittest.TestCase): + """Tests for child profiler CLI argument construction.""" + + def test_build_child_profiler_args_diff_flamegraph(self): + """Test child args use the real --diff-flamegraph flag.""" + args = argparse.Namespace( + sample_interval_usec=1000, + duration=None, + all_threads=False, + realtime_stats=False, + native=False, + gc=True, + opcodes=False, + async_aware=False, + mode="wall", + format="diff_flamegraph", + diff_baseline="baseline.bin", + ) + + child_args = _build_child_profiler_args(args) + + self.assertIn("--diff-flamegraph", child_args) + self.assertNotIn("--diff_flamegraph", child_args) + + flag_index = child_args.index("--diff-flamegraph") + self.assertGreater(len(child_args), flag_index + 1) + self.assertEqual(child_args[flag_index + 1], "baseline.bin") + + @requires_remote_subprocess_debugging() class TestGetChildPids(unittest.TestCase): """Tests for the get_child_pids function.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py index c522c50d1fd5fac..9c0734ac804e1bc 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_cli.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_cli.py @@ -1,6 +1,7 @@ """Tests for sampling profiler CLI argument parsing and functionality.""" import io +import json import os import subprocess import sys @@ -21,9 +22,19 @@ requires_remote_subprocess_debugging, ) -from profiling.sampling.cli import main -from profiling.sampling.constants import PROFILING_MODE_ALL, PROFILING_MODE_WALL +from profiling.sampling.cli import ( + FORMAT_EXTENSIONS, + _create_collector, + _generate_output_filename, + main, +) +from profiling.sampling.constants import ( + PROFILING_MODE_ALL, + PROFILING_MODE_CPU, + PROFILING_MODE_WALL, +) from profiling.sampling.errors import SamplingScriptNotFoundError, SamplingModuleNotFoundError, SamplingUnknownProcessError +from profiling.sampling.jsonl_collector import JsonlCollector class TestSampleProfilerCLI(unittest.TestCase): def _setup_sync_mocks(self, mock_socket, mock_popen): @@ -912,3 +923,65 @@ def test_cli_replay_reader_errors_exit_cleanly(self): str(cm.exception), "Error: Unsupported format version 2", ) + + def test_cli_jsonl_format_mutually_exclusive_with_pstats(self): + """--jsonl and --pstats cannot be combined (mutually exclusive group).""" + with ( + mock.patch( + "sys.argv", + [ + "profiling.sampling.cli", + "attach", + "12345", + "--jsonl", + "--pstats", + ], + ), + mock.patch("sys.stderr", io.StringIO()), + ): + with self.assertRaises(SystemExit): + main() + + def test_cli_jsonl_extension_in_format_extensions(self): + """FORMAT_EXTENSIONS maps 'jsonl' -> 'jsonl' so default filenames work.""" + self.assertEqual(FORMAT_EXTENSIONS["jsonl"], "jsonl") + self.assertEqual(_generate_output_filename("jsonl", 12345), "jsonl_12345.jsonl") + + def test_cli_jsonl_create_collector_propagates_mode(self): + """_create_collector('jsonl', ..., mode=X) lands X in the meta record.""" + collector = _create_collector( + "jsonl", + sample_interval_usec=1000, + skip_idle=False, + mode=PROFILING_MODE_CPU, + ) + self.assertIsInstance(collector, JsonlCollector) + + with tempfile.NamedTemporaryFile(suffix=".jsonl", delete=False) as f: + jsonl_path = f.name + self.addCleanup(os.unlink, jsonl_path) + collector.export(jsonl_path) + with open(jsonl_path, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + meta = next(r for r in records if r["type"] == "meta") + self.assertEqual(meta["mode"], "cpu") + + def test_cli_jsonl_rejects_opcodes_combination(self): + """--opcodes is incompatible with --jsonl per opcodes_compatible_formats.""" + test_args = [ + "profiling.sampling.cli", + "attach", + "12345", + "--jsonl", + "--opcodes", + ] + with ( + mock.patch("sys.argv", test_args), + mock.patch("sys.stderr", io.StringIO()) as mock_stderr, + mock.patch("profiling.sampling.cli.sample"), + self.assertRaises(SystemExit) as cm, + ): + main() + + self.assertEqual(cm.exception.code, 2) + self.assertIn("--opcodes", mock_stderr.getvalue()) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py index 240ec8a195c43b2..1ab31af67fec522 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_collectors.py @@ -11,12 +11,15 @@ try: import _remote_debugging # noqa: F401 + from profiling.sampling import gecko_collector from profiling.sampling.pstats_collector import PstatsCollector from profiling.sampling.stack_collector import ( CollapsedStackCollector, FlamegraphCollector, ) + from profiling.sampling.jsonl_collector import JsonlCollector from profiling.sampling.gecko_collector import GeckoCollector + from profiling.sampling.heatmap_collector import _TemplateLoader from profiling.sampling.collector import extract_lineno, normalize_location from profiling.sampling.opcode_utils import get_opcode_info, format_opcode from profiling.sampling.constants import ( @@ -38,7 +41,7 @@ from test.support import captured_stdout, captured_stderr from .mocks import MockFrameInfo, MockThreadInfo, MockInterpreterInfo, LocationInfo, make_diff_collector_with_mock_baseline -from .helpers import close_and_unlink +from .helpers import close_and_unlink, jsonl_tables def resolve_name(node, strings): @@ -57,6 +60,42 @@ def find_child_by_name(children, strings, substr): return None +def export_gecko_profile(testcase, collector): + gecko_out = tempfile.NamedTemporaryFile(suffix=".json", delete=False) + testcase.addCleanup(close_and_unlink, gecko_out) + # We cannot overwrite an open file on Windows. + gecko_out.close() + + with captured_stdout(), captured_stderr(): + collector.export(gecko_out.name) + + testcase.assertGreater(os.path.getsize(gecko_out.name), 0) + with open(gecko_out.name, encoding="utf-8") as file: + return json.load(file) + + +def assert_gecko_column_lengths(testcase, table, columns): + expected = table["length"] + for column in columns: + testcase.assertEqual( + len(table[column]), expected, + f"{column!r} has wrong length", + ) + + +def gecko_marker_names(profile, markers): + string_array = profile["shared"]["stringArray"] + return [string_array[idx] for idx in markers["name"]] + + +def gecko_opcode_marker_data(profile): + markers = profile["threads"][0]["markers"] + return [ + data for data in markers["data"] + if data.get("type") == "Opcode" + ] + + class TestSampleProfilerComponents(unittest.TestCase): """Unit tests for individual profiler components.""" @@ -81,6 +120,18 @@ def test_mock_frame_info_with_empty_and_unicode_values(self): self.assertEqual(frame.location.lineno, 999999) self.assertEqual(frame.funcname, long_funcname) + def test_heatmap_navigation_restarts_line_highlight(self): + """Test heatmap navigation can replay target line highlights.""" + loader = _TemplateLoader() + + self.assertIn(".code-line:target", loader.file_css) + self.assertIn("function restartLineHighlight(target)", loader.file_js) + self.assertIn("target.style.animation = 'none'", loader.file_js) + self.assertIn("void target.offsetWidth", loader.file_js) + self.assertIn("url.href === window.location.href", loader.file_js) + self.assertIn("navigateToLine(JSON.parse(navData).link)", loader.file_js) + self.assertIn("navigateToLine(linkData.link)", loader.file_js) + def test_pstats_collector_with_extreme_intervals_and_empty_data(self): """Test PstatsCollector handles zero/large intervals, empty frames, None thread IDs, and duplicate frames.""" # Test with zero interval @@ -569,9 +620,10 @@ def test_gecko_collector_basic(self): # Verify samples samples = thread_data["samples"] - self.assertEqual(len(samples["stack"]), 1) - self.assertEqual(len(samples["time"]), 1) self.assertEqual(samples["length"], 1) + assert_gecko_column_lengths( + self, samples, ("stack", "time", "eventDelay") + ) # Verify function table structure and content func_table = thread_data["funcTable"] @@ -608,9 +660,6 @@ def test_gecko_collector_basic(self): @unittest.skipIf(is_emscripten, "threads not available") def test_gecko_collector_export(self): """Test Gecko profile export functionality.""" - gecko_out = tempfile.NamedTemporaryFile(suffix=".json", delete=False) - self.addCleanup(close_and_unlink, gecko_out) - collector = GeckoCollector(1000) test_frames1 = [ @@ -643,17 +692,7 @@ def test_gecko_collector_export(self): collector.collect(test_frames2) collector.collect(test_frames3) - # Export gecko profile - with captured_stdout(), captured_stderr(): - collector.export(gecko_out.name) - - # Verify file was created and contains valid data - self.assertTrue(os.path.exists(gecko_out.name)) - self.assertGreater(os.path.getsize(gecko_out.name), 0) - - # Check file contains valid JSON - with open(gecko_out.name, "r") as f: - profile_data = json.load(f) + profile_data = export_gecko_profile(self, collector) # Should be valid Gecko profile format self.assertIn("meta", profile_data) @@ -674,6 +713,100 @@ def test_gecko_collector_export(self): self.assertIn("func2", string_array) self.assertIn("other_func", string_array) + thread_data = profile_data["threads"][0] + assert_gecko_column_lengths( + self, thread_data["samples"], ("stack", "time", "eventDelay") + ) + + @unittest.skipIf(is_emscripten, "threads not available") + def test_gecko_collector_export_after_spill_flush(self): + """Test Gecko profile export after spill buffers flush to disk.""" + old_buffer_bytes = gecko_collector.DEFAULT_SPILL_BUFFER_BYTES + gecko_collector.DEFAULT_SPILL_BUFFER_BYTES = 1 + self.addCleanup( + setattr, gecko_collector, "DEFAULT_SPILL_BUFFER_BYTES", + old_buffer_bytes + ) + + collector = GeckoCollector(1000) + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("file.py", 10, "func")], + status=THREAD_STATUS_HAS_GIL, + ) + ], + ) + ] + collector.collect(test_frames, timestamps_us=[1000, 2000, 3000]) + + profile_data = export_gecko_profile(self, collector) + samples = profile_data["threads"][0]["samples"] + self.assertEqual(samples["length"], 3) + assert_gecko_column_lengths( + self, samples, ("stack", "time", "eventDelay") + ) + + @unittest.skipIf(is_emscripten, "threads not available") + def test_gecko_collector_rejects_collect_after_export(self): + collector = GeckoCollector(1000) + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("file.py", 10, "func")], + status=THREAD_STATUS_HAS_GIL, + ) + ], + ) + ] + collector.collect(test_frames) + export_gecko_profile(self, collector) + + with self.assertRaisesRegex(RuntimeError, "after export"): + collector.collect(test_frames) + + @unittest.skipIf(is_emscripten, "threads not available") + def test_gecko_collector_export_failure_keeps_existing_file(self): + collector = GeckoCollector(1000) + test_frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("file.py", 10, "func")], + status=THREAD_STATUS_HAS_GIL, + ) + ], + ) + ] + collector.collect(test_frames) + + with tempfile.TemporaryDirectory() as temp_dir: + filename = os.path.join(temp_dir, "profile.json") + with open(filename, "w", encoding="utf-8") as file: + file.write("existing") + + before = set(os.listdir(temp_dir)) + + def fail(file): + raise OSError("boom") + + collector._stream_profile = fail + with captured_stdout(), captured_stderr(): + with self.assertRaisesRegex(OSError, "boom"): + collector.export(filename) + + with open(filename, encoding="utf-8") as file: + self.assertEqual(file.read(), "existing") + self.assertEqual(set(os.listdir(temp_dir)), before) + def test_gecko_collector_markers(self): """Test Gecko profile markers for GIL and CPU state tracking.""" collector = GeckoCollector(1000) @@ -757,21 +890,16 @@ def test_gecko_collector_markers(self): self.assertIn("markers", thread_data) markers = thread_data["markers"] - # Should have marker arrays - self.assertIn("name", markers) - self.assertIn("startTime", markers) - self.assertIn("endTime", markers) - self.assertIn("category", markers) self.assertGreater( markers["length"], 0, "Should have generated markers" ) - - # Get marker names from string table - string_array = profile_data["shared"]["stringArray"] - marker_names = [string_array[idx] for idx in markers["name"]] + assert_gecko_column_lengths( + self, markers, + ("data", "name", "startTime", "endTime", "phase", "category"), + ) # Verify we have different marker types - marker_name_set = set(marker_names) + marker_name_set = set(gecko_marker_names(profile_data, markers)) # Should have "Has GIL" markers (when thread had GIL) self.assertIn( @@ -1402,6 +1530,39 @@ def test_diff_flamegraph_elided_stacks(self): self.assertGreater(child["baseline"], 0) self.assertAlmostEqual(child["diff"], -child["baseline"]) + def test_diff_flamegraph_elided_top_level_root(self): + """Elided top-level roots do not crash metadata generation.""" + baseline_frames_1 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 10, "kept_leaf"), + MockFrameInfo("file.py", 20, "kept_root"), + ]) + ]) + ] + baseline_frames_2 = [ + MockInterpreterInfo(0, [ + MockThreadInfo(1, [ + MockFrameInfo("file.py", 30, "old_leaf"), + MockFrameInfo("file.py", 40, "old_root"), + ]) + ]) + ] + + diff = make_diff_collector_with_mock_baseline([ + baseline_frames_1, + baseline_frames_2, + ]) + diff.collect(baseline_frames_1) + + data = diff._convert_to_flamegraph_format() + elided = data["stats"]["elided_flamegraph"] + elided_strings = elided.get("strings", []) + children = elided.get("children", []) + + self.assertEqual(len(children), 1) + self.assertIn("old_root", resolve_name(children[0], elided_strings)) + def test_diff_flamegraph_function_matched_despite_line_change(self): """Functions match by (filename, funcname), ignoring lineno.""" baseline_frames = [ @@ -1669,6 +1830,393 @@ def test_diff_flamegraph_load_baseline(self): self.assertAlmostEqual(cold_node["diff"], -1.0) self.assertAlmostEqual(cold_node["diff_pct"], -50.0) + def test_jsonl_collector_export_exact_output(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + collector.run_id = "run-123" + + test_frames1 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] + test_frames2 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("file.py", 10, "func1"), + MockFrameInfo("file.py", 20, "func2"), + ], + ) + ], + ) + ] # Same stack + test_frames3 = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("other.py", 5, "other_func")] + ) + ], + ) + ] + + collector.collect(test_frames1) + collector.collect(test_frames2) + collector.collect(test_frames3) + + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + content = f.read() + + self.assertEqual( + content, + ( + '{"type":"meta","v":0,"run_id":"run-123","sample_interval_usec":1000}\n' + '{"type":"string_table","v":0,"run_id":"run-123","strings":[{"str_id":0,"value":"func1"},{"str_id":1,"value":"file.py"},{"str_id":2,"value":"func2"},{"str_id":3,"value":"other_func"},{"str_id":4,"value":"other.py"}]}\n' + '{"type":"frame_table","v":0,"run_id":"run-123","frames":[{"frame_id":0,"path_str_id":1,"func_str_id":0,"line":10,"end_line":10},{"frame_id":1,"path_str_id":1,"func_str_id":2,"line":20,"end_line":20},{"frame_id":2,"path_str_id":4,"func_str_id":3,"line":5,"end_line":5}]}\n' + '{"type":"agg","v":0,"run_id":"run-123","kind":"frame","scope":"final","samples_total":3,"entries":[{"frame_id":0,"self":2,"cumulative":2},{"frame_id":1,"self":0,"cumulative":2},{"frame_id":2,"self":1,"cumulative":1}]}\n' + '{"type":"end","v":0,"run_id":"run-123","samples_total":3}\n' + ), + ) + + def test_jsonl_collector_export_includes_mode_in_meta(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000, mode=PROFILING_MODE_CPU) + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, [MockFrameInfo("file.py", 10, "func")] + ) + ], + ) + ] + ) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + meta_record = next( + record for record in records if record["type"] == "meta" + ) + self.assertEqual(meta_record["mode"], "cpu") + + def test_jsonl_collector_export_empty_profile(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + collector.run_id = "run-123" + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + self.assertEqual( + [record["type"] for record in records], ["meta", "end"] + ) + self.assertEqual(records[0]["sample_interval_usec"], 1000) + self.assertEqual(records[0]["run_id"], "run-123") + self.assertEqual(records[1]["samples_total"], 0) + self.assertEqual(records[1]["run_id"], "run-123") + + def test_jsonl_collector_recursive_frames_counted_once_per_sample(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo( + "recursive.py", 10, "recursive_func" + ), + MockFrameInfo( + "recursive.py", 10, "recursive_func" + ), + MockFrameInfo( + "recursive.py", 10, "recursive_func" + ), + ], + ) + ], + ) + ] + ) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, _, frame_defs, agg_record, end_record = jsonl_tables(records) + self.assertEqual(len(frame_defs), 1) + self.assertEqual( + agg_record["entries"], + [ + { + "frame_id": frame_defs[0]["frame_id"], + "self": 1, + "cumulative": 1, + } + ], + ) + self.assertEqual(agg_record["samples_total"], 1) + self.assertEqual(end_record["samples_total"], 1) + + def test_jsonl_collector_skip_idle_filters_threads(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + active_status = THREAD_STATUS_HAS_GIL | THREAD_STATUS_ON_CPU + frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [MockFrameInfo("active1.py", 10, "active_func1")], + status=active_status, + ), + MockThreadInfo( + 2, + [MockFrameInfo("idle.py", 20, "idle_func")], + status=0, + ), + MockThreadInfo( + 3, + [MockFrameInfo("active2.py", 30, "active_func2")], + status=active_status, + ), + ], + ) + ] + + def export_summary(skip_idle): + collector = JsonlCollector(1000, skip_idle=skip_idle) + collector.collect(frames) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, str_defs, frame_defs, agg_record, _ = jsonl_tables(records) + paths = {str_defs[item["path_str_id"]] for item in frame_defs} + funcs = {str_defs[item["func_str_id"]] for item in frame_defs} + return paths, funcs, agg_record["samples_total"] + + paths, funcs, samples_total = export_summary(skip_idle=True) + self.assertEqual(paths, {"active1.py", "active2.py"}) + self.assertEqual(funcs, {"active_func1", "active_func2"}) + self.assertEqual(samples_total, 2) + + paths, funcs, samples_total = export_summary(skip_idle=False) + self.assertEqual(paths, {"active1.py", "idle.py", "active2.py"}) + self.assertEqual(funcs, {"active_func1", "idle_func", "active_func2"}) + self.assertEqual(samples_total, 3) + + def test_jsonl_collector_splits_large_exports_into_chunks(self): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + + for i in range(257): + collector.collect( + [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo( + f"file{i}.py", i + 1, f"func{i}" + ) + ], + ) + ], + ) + ] + ) + + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + run_ids = {record["run_id"] for record in records} + self.assertEqual(len(run_ids), 1) + self.assertRegex(next(iter(run_ids)), r"^[0-9a-f]{32}$") + + _, str_defs, frame_defs, agg_record, end_record = jsonl_tables( + records + ) + str_chunks = [ + record for record in records if record["type"] == "string_table" + ] + frame_chunks = [ + record for record in records if record["type"] == "frame_table" + ] + agg_chunks = [record for record in records if record["type"] == "agg"] + + self.assertEqual( + [len(record["strings"]) for record in str_chunks], + [256, 256, 2], + ) + self.assertEqual( + [len(record["frames"]) for record in frame_chunks], [256, 1] + ) + self.assertEqual( + [len(record["entries"]) for record in agg_chunks], [256, 1] + ) + self.assertEqual(len(str_defs), 514) + self.assertEqual(len(frame_defs), 257) + self.assertEqual(agg_record["samples_total"], 257) + self.assertEqual(end_record["samples_total"], 257) + + def test_jsonl_collector_respects_weight_for_rle_batched_samples(self): + """weight>1 (from binary replay RLE) is honored in self/cumulative.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + leaf = MockFrameInfo("file.py", 10, "leaf") + non_leaf = MockFrameInfo("file.py", 20, "non_leaf") + + collector.process_frames([leaf, non_leaf], _thread_id=1, weight=5) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, str_defs, frame_defs, agg, end = jsonl_tables(records) + self.assertEqual(end["samples_total"], 5) + self.assertEqual(agg["samples_total"], 5) + self.assertEqual( + {str_defs[fd["func_str_id"]]: fd["frame_id"] for fd in frame_defs}, + {"leaf": 0, "non_leaf": 1}, + ) + self.assertEqual(agg["entries"], [ + {"frame_id": 0, "self": 5, "cumulative": 5}, + {"frame_id": 1, "self": 0, "cumulative": 5}, + ]) + + def test_jsonl_collector_recursion_with_weight(self): + """Recursion dedup respects weight, not occurrence count.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + recursive = MockFrameInfo("rec.py", 10, "f") + + collector.process_frames([recursive] * 3, _thread_id=1, weight=3) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, _, frame_defs, agg, _ = jsonl_tables(records) + self.assertEqual(len(frame_defs), 1) + self.assertEqual(agg["entries"], [ + {"frame_id": 0, "self": 3, "cumulative": 3}, + ]) + + def test_jsonl_collector_emits_col_and_end_col_when_present(self): + """All four location fields are emitted when col/end_col are >= 0.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + frame = MockFrameInfo("test.py", 0, "f") + frame.location = LocationInfo(42, 45, 4, 12) + frames = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [frame], status=THREAD_STATUS_HAS_GIL)] + ) + ] + collector.collect(frames) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, str_defs, frame_defs, _, _ = jsonl_tables(records) + self.assertEqual(frame_defs, [ + { + "frame_id": 0, + "path_str_id": 1, + "func_str_id": 0, + "line": 42, + "end_line": 45, + "col": 4, + "end_col": 12, + }, + ]) + self.assertEqual(str_defs, {0: "f", 1: "test.py"}) + + def test_jsonl_collector_partial_location_elision(self): + """Negative col/end_col/end_line fields are individually elided.""" + # _get_or_create_frame_id interns funcname before filename, so + # func_str_id=0 ("f") and path_str_id=1 ("test.py"). + common = {"frame_id": 0, "path_str_id": 1, "func_str_id": 0} + cases = [ + (LocationInfo(42, 45, -1, 12), + {**common, "line": 42, "end_line": 45, "end_col": 12}), + (LocationInfo(42, 45, 4, -1), + {**common, "line": 42, "end_line": 45, "col": 4}), + (LocationInfo(42, 0, 4, 8), + {**common, "line": 42, "col": 4, "end_col": 8}), + ] + for loc, expected_frame_def in cases: + with self.subTest(location=loc): + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(1000) + frame = MockFrameInfo("test.py", 0, "f") + frame.location = loc + frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [frame], status=THREAD_STATUS_HAS_GIL)], + ) + ] + collector.collect(frames) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, _, frame_defs, _, _ = jsonl_tables(records) + self.assertEqual(frame_defs, [expected_frame_def]) + class TestRecursiveFunctionHandling(unittest.TestCase): """Tests for correct handling of recursive functions in cumulative stats.""" @@ -1878,6 +2426,20 @@ def test_extract_lineno_from_none(self): """Test extracting lineno from None (synthetic frames).""" self.assertEqual(extract_lineno(None), 0) + def test_extract_lineno_from_int(self): + """Test extracting lineno from a bare integer line number. + + Mirrors normalize_location's int contract so callers like the + collapsed/flamegraph collectors do not crash on a bare-int location. + """ + self.assertEqual(extract_lineno(42), 42) + self.assertEqual(extract_lineno(0), 0) + + def test_normalize_location_with_int(self): + """Test normalize_location expands a legacy integer line number.""" + result = normalize_location(42) + self.assertEqual(result, (42, 42, -1, -1)) + def test_normalize_location_with_location_info(self): """Test normalize_location passes through LocationInfo.""" loc = LocationInfo(10, 15, 0, 5) @@ -2068,6 +2630,85 @@ def test_gecko_collector_with_location_info(self): # Verify function name is in string table self.assertIn("handle_request", string_array) + def test_jsonl_collector_with_location_info(self): + """Test JsonlCollector handles LocationInfo properly.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(sample_interval_usec=1000) + + # Frame with LocationInfo + frame = MockFrameInfo("test.py", 42, "my_function") + frames = [ + MockInterpreterInfo( + 0, [MockThreadInfo(1, [frame], status=THREAD_STATUS_HAS_GIL)] + ) + ] + collector.collect(frames) + + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + meta, str_defs, frame_defs, agg, end = jsonl_tables(records) + self.assertEqual(meta["sample_interval_usec"], 1000) + self.assertEqual(agg["samples_total"], 1) + self.assertEqual(end["samples_total"], 1) + self.assertEqual(len(frame_defs), 1) + self.assertEqual(str_defs[frame_defs[0]["path_str_id"]], "test.py") + self.assertEqual(str_defs[frame_defs[0]["func_str_id"]], "my_function") + self.assertEqual( + frame_defs[0], + { + "frame_id": 0, + "path_str_id": frame_defs[0]["path_str_id"], + "func_str_id": frame_defs[0]["func_str_id"], + "line": 42, + "end_line": 42, + }, + ) + + def test_jsonl_collector_with_none_location(self): + """Test JsonlCollector handles None location (synthetic frames).""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + collector = JsonlCollector(sample_interval_usec=1000) + + # Create frame with None location (like GC frame) + frame = MockFrameInfo("~", 0, "") + frame.location = None # Synthetic frame has no location + frames = [ + MockInterpreterInfo( + 0, + [MockThreadInfo(1, [frame], status=THREAD_STATUS_HAS_GIL)] + ) + ] + collector.collect(frames) + + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + meta, str_defs, frame_defs, agg, end = jsonl_tables(records) + self.assertEqual(meta["sample_interval_usec"], 1000) + self.assertEqual(agg["samples_total"], 1) + self.assertEqual(end["samples_total"], 1) + self.assertEqual(len(frame_defs), 1) + self.assertEqual(str_defs[frame_defs[0]["path_str_id"]], "~") + self.assertEqual(str_defs[frame_defs[0]["func_str_id"]], "") + self.assertEqual( + frame_defs[0], + { + "frame_id": 0, + "path_str_id": frame_defs[0]["path_str_id"], + "func_str_id": frame_defs[0]["func_str_id"], + "line": 0, + }, + ) + class TestOpcodeHandling(unittest.TestCase): """Tests for opcode field handling in collectors.""" @@ -2132,6 +2773,7 @@ def test_gecko_collector_opcodes_enabled(self): def test_gecko_opcode_state_tracking(self): """Test that GeckoCollector tracks opcode state changes.""" collector = GeckoCollector(sample_interval_usec=1000, opcodes=True) + self.addCleanup(collector._cleanup_spills) # First sample with opcode 90 (RAISE_VARARGS) frame1 = MockFrameInfo("test.py", 10, "func", opcode=90) @@ -2175,10 +2817,28 @@ def test_gecko_opcode_state_change_emits_marker(self): collector.collect(frames2) # Should have emitted a marker for the first opcode - thread_data = collector.threads[1] - markers = thread_data["markers"] - # At least one marker should have been added - self.assertGreater(len(markers["name"]), 0) + profile = collector._build_profile() + markers = profile["threads"][0]["markers"] + assert_gecko_column_lengths( + self, markers, + ("data", "name", "startTime", "endTime", "phase", "category"), + ) + opcode_markers = gecko_opcode_marker_data(profile) + self.assertIn( + { + "opcode": 90, + "line": 10, + "function": "func", + }, + [ + { + "opcode": marker["opcode"], + "line": marker["line"], + "function": marker["function"], + } + for marker in opcode_markers + ], + ) def test_gecko_opcode_markers_not_emitted_when_disabled(self): """Test that no opcode markers when opcodes=False.""" @@ -2202,8 +2862,9 @@ def test_gecko_opcode_markers_not_emitted_when_disabled(self): ] collector.collect(frames2) - # opcode_state should not be tracked - self.assertEqual(len(collector.opcode_state), 0) + profile = collector._build_profile() + self.assertEqual(gecko_opcode_marker_data(profile), []) + self.assertEqual(profile["meta"]["markerSchema"], []) def test_gecko_opcode_with_none_opcode(self): """Test that None opcode doesn't cause issues.""" @@ -2219,9 +2880,8 @@ def test_gecko_opcode_with_none_opcode(self): ] collector.collect(frames) - # Should track the state but opcode is None - self.assertIn(1, collector.opcode_state) - self.assertIsNone(collector.opcode_state[1][0]) + profile = collector._build_profile() + self.assertEqual(gecko_opcode_marker_data(profile), []) class TestCollectorFrameFormat(unittest.TestCase): @@ -2288,6 +2948,28 @@ def test_gecko_collector_frame_format(self): # Should have recorded 3 functions self.assertEqual(thread["funcTable"]["length"], 3) + def test_jsonl_collector_frame_format(self): + """Test JsonlCollector with 4-element frame format.""" + collector = JsonlCollector(sample_interval_usec=1000) + collector.collect(self._make_sample_frames()) + + with tempfile.NamedTemporaryFile(delete=False) as f: + self.addClassCleanup(close_and_unlink, f) + collector.export(f.name) + + with open(f.name, "r", encoding="utf-8") as fp: + records = [json.loads(line) for line in fp] + + _, str_defs, frame_defs, _, _ = jsonl_tables(records) + + self.assertEqual(len(frame_defs), 3) + + paths = {str_defs[item["path_str_id"]] for item in frame_defs} + funcs = {str_defs[item["func_str_id"]] for item in frame_defs} + + self.assertEqual(paths, {"app.py", "utils.py", "lib.py"}) + self.assertEqual(funcs, {"main", "helper", "process"}) + class TestInternalFrameFiltering(unittest.TestCase): """Tests for filtering internal profiler frames from output.""" @@ -2415,3 +3097,42 @@ def test_collapsed_stack_collector_filters_internal_frames(self): for (call_tree, _), _ in collector.stack_counter.items(): for filename, _, _ in call_tree: self.assertNotIn("_sync_coordinator", filename) + + def test_jsonl_collector_filters_internal_frames(self): + """Test that JsonlCollector filters out internal frames.""" + jsonl_out = tempfile.NamedTemporaryFile(delete=False) + self.addCleanup(close_and_unlink, jsonl_out) + + frames = [ + MockInterpreterInfo( + 0, + [ + MockThreadInfo( + 1, + [ + MockFrameInfo("app.py", 50, "run"), + MockFrameInfo("/lib/_sync_coordinator.py", 100, "main"), + MockFrameInfo("", 87, "_run_code"), + ], + status=THREAD_STATUS_HAS_GIL, + ) + ], + ) + ] + + collector = JsonlCollector(sample_interval_usec=1000) + collector.collect(frames) + collector.export(jsonl_out.name) + + with open(jsonl_out.name, "r", encoding="utf-8") as f: + records = [json.loads(line) for line in f] + + _, str_defs, frame_defs, _, _ = jsonl_tables(records) + + paths = {str_defs[item["path_str_id"]] for item in frame_defs} + + self.assertIn("app.py", paths) + self.assertIn("", paths) + + for path in paths: + self.assertNotIn("_sync_coordinator", path) diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py index c6731e956391a91..3487647b76683ee 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_integration.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_integration.py @@ -686,8 +686,6 @@ def test_script_error_treatment(self): "-m", "profiling.sampling.cli", "run", - "-d", - "1", script_file.name, ], capture_output=True, diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py index c0d39f487c8cbdf..ae4ee969f34d8b3 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_live_collector_ui.py @@ -825,18 +825,34 @@ def test_get_all_lines_full_display(self): class TestLiveModeErrors(unittest.TestCase): """Tests running error commands in the live mode fails gracefully.""" + class QuitWhenFinishedDisplay(MockDisplay): + def __init__(self, collector): + super().__init__() + self.collector = collector + + def get_input(self): + ch = super().get_input() + if ch != -1: + return ch + # Sampling only stops once the target process has exited, at + # which point the collector is marked finished. Quit then so the + # run can surface the target's stderr. We must not rely on the + # target's pid still being signalable: once it exits it lingers + # as a zombie (it is reaped after sample_live returns), so a + # liveness check would never observe it gone and would hang. + if self.collector.finished: + return ord('q') + return -1 + def mock_curses_wrapper(self, func): func(mock.MagicMock()) def mock_init_curses_side_effect(self, n_times, mock_self, stdscr): - mock_self.display = MockDisplay() - # Allow the loop to run for a bit (approx 0.5s) before quitting - # This ensures we don't exit too early while the subprocess is - # still failing + mock_self.display = self.QuitWhenFinishedDisplay(mock_self) + # Feed non-input events so live mode keeps polling while the target + # process is still running; once it exits the display quits on its own. for _ in range(n_times): mock_self.display.simulate_input(-1) - if n_times >= 500: - mock_self.display.simulate_input(ord('q')) def test_run_failed_module_live(self): """Test that running a existing module that fails exits with clean error.""" diff --git a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py index 68bc59a5414a05c..2f5a5e273286590 100644 --- a/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py +++ b/Lib/test/test_profiling/test_sampling_profiler/test_profiler.py @@ -198,8 +198,83 @@ def test_sample_profiler_sample_method_timing(self): self.assertIn("samples", result) # Verify collector was called multiple times - self.assertGreaterEqual(mock_collector.collect.call_count, 5) - self.assertLessEqual(mock_collector.collect.call_count, 11) + total_weight = sum( + len(c.kwargs.get("timestamps_us") or [None]) + for c in mock_collector.collect.call_args_list + ) + self.assertGreaterEqual(total_weight, 5) + self.assertLessEqual(total_weight, 11) + + def test_sample_profiler_does_not_buffer_non_aggregating_collectors(self): + """Test that non-aggregating collectors get each sample immediately.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = False + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + manager = mock.Mock() + manager.attach_mock(u.instance.get_stack_trace, "unwind") + manager.attach_mock(mock_collector.collect, "collect") + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [0.0, 0.01, 0.011, 0.02, 0.03] + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.025) + + self.assertEqual( + manager.mock_calls, + [ + mock.call.unwind(), + mock.call.collect(stack_frames), + mock.call.unwind(), + mock.call.collect(stack_frames), + ], + ) + + def test_sample_profiler_flushes_aggregated_batches_at_limit(self): + """Test that aggregating collectors flush after MAX_PENDING_SAMPLES samples.""" + + stack_frames = [mock.sentinel.stack_frames] + mock_collector = mock.MagicMock() + mock_collector.aggregating = True + + with self._patched_unwinder() as u: + u.instance.get_stack_trace.return_value = stack_frames + + profiler = SampleProfiler( + pid=12345, sample_interval_usec=10000, all_threads=False + ) + + times = [ + 0.0, + 0.01, 0.011, + 0.02, 0.021, + 0.03, 0.031, + 0.04, 0.041, + 0.05, 0.051, + ] + with mock.patch("profiling.sampling.sample.MAX_PENDING_SAMPLES", 2): + with mock.patch("time.perf_counter", side_effect=times): + with io.StringIO() as output: + with mock.patch("sys.stdout", output): + profiler.sample(mock_collector, duration_sec=0.045) + + batches = [ + (c.args[0], len(c.kwargs["timestamps_us"])) + for c in mock_collector.collect.call_args_list + ] + self.assertEqual( + batches, + [(stack_frames, 2), (stack_frames, 2), (stack_frames, 1)], + ) def test_sample_profiler_error_handling(self): """Test that the sample method handles errors gracefully.""" diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py index aa090b464a72222..bdf57776c82be13 100644 --- a/Lib/test/test_pwd.py +++ b/Lib/test/test_pwd.py @@ -1,3 +1,5 @@ +import random +import string import sys import unittest from test.support import import_helper @@ -56,59 +58,57 @@ def test_values_extended(self): def test_errors(self): self.assertRaises(TypeError, pwd.getpwuid) self.assertRaises(TypeError, pwd.getpwuid, 3.14) + self.assertRaises(TypeError, pwd.getpwuid, 0.0) + self.assertRaises(TypeError, pwd.getpwuid, 0, 0) + # should be out of uid_t range + self.assertRaises(KeyError, pwd.getpwuid, 2**128) + self.assertRaises(KeyError, pwd.getpwuid, -2**128) self.assertRaises(TypeError, pwd.getpwnam) self.assertRaises(TypeError, pwd.getpwnam, 42) - self.assertRaises(TypeError, pwd.getpwall, 42) + self.assertRaises(TypeError, pwd.getpwnam, b'root') + self.assertRaises(TypeError, pwd.getpwnam, 'root', 0) # embedded null character self.assertRaisesRegex(ValueError, 'null', pwd.getpwnam, 'a\x00b') + self.assertRaisesRegex(ValueError, 'null', pwd.getpwnam, 'root\x00') + self.assertRaises(UnicodeEncodeError, pwd.getpwnam, 'roo\udc74') + self.assertRaises(KeyError, pwd.getpwnam, '') + self.assertRaises(TypeError, pwd.getpwall, 42) - # try to get some errors - bynames = {} - byuids = {} - for (n, p, u, g, gecos, d, s) in pwd.getpwall(): - bynames[n] = u - byuids[u] = n - - allnames = list(bynames.keys()) - namei = 0 - fakename = allnames[namei] if allnames else "invaliduser" - while fakename in bynames: - chars = list(fakename) - for i in range(len(chars)): - if chars[i] == 'z': - chars[i] = 'A' - break - elif chars[i] == 'Z': - continue - else: - chars[i] = chr(ord(chars[i]) + 1) - break - else: - namei = namei + 1 - try: - fakename = allnames[namei] - except IndexError: - # should never happen... if so, just forget it - break - fakename = ''.join(chars) - - self.assertRaises(KeyError, pwd.getpwnam, fakename) - - # In some cases, byuids isn't a complete list of all users in the - # system, so if we try to pick a value not in byuids (via a perturbing - # loop, say), pwd.getpwuid() might still be able to find data for that - # uid. Using sys.maxint may provoke the same problems, but hopefully - # it will be a more repeatable failure. - fakeuid = sys.maxsize - self.assertNotIn(fakeuid, byuids) - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + # Find a non-existent user name. + # getpwall() will not necessarily report all existing users + # (typical for LDAP based directories in big organizations). + for _ in range(30): + fakename = ''.join(random.choices(string.ascii_lowercase, k=6)) + try: + pwd.getpwnam(fakename) + except KeyError: + break + else: + self.fail('Cannot find non-existent user name') + + # Find a non-existent uid. + maxuid = max(e.pw_uid for e in pwd.getpwall()) + if maxuid < 2**15: + maxuid = 2**15 + elif maxuid < 2**16: + maxuid = 2**16-1 + else: + maxuid = 2**31 + for _ in range(30): + fakeuid = random.randrange(maxuid) + try: + pwd.getpwuid(fakeuid) + except KeyError: + break + else: + self.fail('Cannot find non-existent uid') + + # On Cygwin, getpwuid(-1) returns 'Unknown+User' user + if sys.platform != 'cygwin': + # -1 shouldn't be a valid uid because it has a special meaning in many + # uid-related functions + self.assertRaises(KeyError, pwd.getpwuid, -1) - # -1 shouldn't be a valid uid because it has a special meaning in many - # uid-related functions - self.assertRaises(KeyError, pwd.getpwuid, -1) - # should be out of uid_t range - self.assertRaises(KeyError, pwd.getpwuid, 2**128) - self.assertRaises(KeyError, pwd.getpwuid, -2**128) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index da2d630d7ace7bb..09113e983fcd94d 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -327,6 +327,12 @@ def test_bad_syntax_with_quiet(self): self.assertEqual(stdout, b'') self.assertEqual(stderr, b'') + def test_bad_syntax_stderr_ends_with_newline(self): + with open(self.source_path, 'w') as file: + file.write(' print("hello world")\n') + rc, stdout, stderr = self.pycompilecmd_failure(self.source_path) + self.assertTrue(stderr.endswith(b'\n')) + def test_file_not_exists(self): should_not_exists = os.path.join(os.path.dirname(__file__), 'should_not_exists.py') rc, stdout, stderr = self.pycompilecmd_failure(self.source_path, should_not_exists) diff --git a/Lib/test/test_pydoc/longsummary.py b/Lib/test/test_pydoc/longsummary.py new file mode 100644 index 000000000000000..0e566d33454119b --- /dev/null +++ b/Lib/test/test_pydoc/longsummary.py @@ -0,0 +1,29 @@ +class C: + """This is a class summary that consists of a very long single line, exceeding the recommended PEP 8 limit. + + The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + """ + def meth(self): + """This is a method summary that consists of a very long single line, exceeding the recommended PEP 8 limit. + + The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + """ + + @property + def prop(self): + """This is a property summary that consists of a very long single line, exceeding the recommended PEP 8 limit. + + The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + """ + +def func(self): + """This is a function summary that consists of a very long single line, exceeding the recommended PEP 8 limit. + + The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + """ + +data = C() +data.__doc__ = """This is a data summary that consists of a very long single line, exceeding the recommended PEP 8 limit. + +The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. +""" diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 2e190d1b81be8ec..25f94c4c740e1ea 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1245,6 +1245,71 @@ def function_with_really_long_name_so_annotations_can_be_rather_small( lambda very_long_parameter_name_that_should_not_fit_into_a_single_line, second_very_long_parameter_name ''' % __name__) + @requires_docstrings + def test_long_summaries(self): + from . import longsummary + doc = pydoc.render_doc(longsummary) + doc = clean_text(doc) + self.assertEqual(doc, '''Python Library Documentation: module test.test_pydoc.longsummary in test.test_pydoc + +NAME + test.test_pydoc.longsummary + +CLASSES + builtins.object + C + + class C(builtins.object) + | This is a class summary that consists of a very long single line, + | exceeding the recommended PEP 8 limit. + | + | The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + | + | Methods defined here: + | + | meth(self) + | This is a method summary that consists of a very long single line, + | exceeding the recommended PEP 8 limit. + | + | The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + | + | ---------------------------------------------------------------------- + | Readonly properties defined here: + | + | prop + | This is a property summary that consists of a very long single line, + | exceeding the recommended PEP 8 limit. + | + | The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + | + | ---------------------------------------------------------------------- + | Data descriptors defined here: + | + | __dict__ + | dictionary for instance variables + | + | __weakref__ + | list of weak references to the object + +FUNCTIONS + func(self) + This is a function summary that consists of a very long single line, + exceeding the recommended PEP 8 limit. + + The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + +DATA + data = + This is a data summary that consists of a very long single line, + exceeding the recommended PEP 8 limit. + + The rest of the docstring body, separated from the summary by a blank line, can also contain very long lines. + +FILE + %s + +''' % inspect.getabsfile(longsummary)) + def test__future__imports(self): # __future__ features are excluded from module help, # except when it's the __future__ module itself @@ -2172,7 +2237,7 @@ def mock_getline(prompt): def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), - sorted(keyword.kwlist)) + sorted(keyword.kwlist + ['case', 'match', 'lazy'])) def test_interact_empty_line_continues(self): # gh-138568: test pressing Enter without input should continue in help session diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index aaa91aca36e3c4c..42309dd8f6c1903 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -227,8 +227,7 @@ def _verify_parse_output(self, operations): "Character data: '\xb5'", "End element: 'root'", ] - for operation, expected_operation in zip(operations, expected_operations): - self.assertEqual(operation, expected_operation) + self.assertEqual(operations, expected_operations) def test_parse_bytes(self): out = self.Outputter() @@ -276,6 +275,168 @@ def test_parse_again(self): self.assertEqual(expat.ErrorString(cm.exception.code), expat.errors.XML_ERROR_FINISHED) + @support.subTests('encoding', [ + # built-in Expat encodings + 'iso-8859-1', 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', + # 8-bit Python encodings + 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', + 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', + 'cp437', 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', + 'cp855', 'cp856', 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', + 'cp863', 'cp865', 'cp866', 'cp869', 'cp874', 'cp1006', 'cp1125', + 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', + 'cp1256', 'cp1257', 'cp1258', + 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', + 'mac-roman', 'mac-turkish', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', + # multi-byte Python encodings + "cp932", "cp949", "cp950", + "Big5","EUC-JP", "GB2312", "GBK", "johab", "Shift_JIS", + 'UTF8', 'utf-8-sig', + "Big5-HKSCS", "EUC_JIS-2004", "EUC_JISX0213", + "Shift_JIS-2004", "Shift_JISX0213", + ]) + def test_supported_encodings(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + data = (f'\n' + f'{c}').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Start element: 'root' {}", + f'Character data: {c!r}', + "End element: 'root'", + ]) + + @support.subTests('encoding', [ + 'UTF-8', 'utf-8', 'utf8', 'utf-16', 'utf-16le', 'utf-16be', + 'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic', + ]) + def test_supported_encodings2(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + data = (f'\n' + '' + '<корінь атрибут="значення">зміст').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Comment: ' коментар '", + "Start element: 'корінь' {'атрибут': 'значення'}", + "Character data: 'зміст'", + "End element: 'корінь'", + ]) + + @support.subTests('encoding', [ + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', + ]) + def test_supported_non_bmp(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + c = '\U00020e6d\U00028e36' + data = (f'\n' + f'{c}').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Start element: 'root' {}", + f'Character data: {c!r}', + "End element: 'root'", + ]) + + @support.subTests('encoding', [ + 'UTF8', 'utf-8-sig', + "Big5-HKSCS", "EUC_JIS-2004", "EUC_JISX0213", + "Shift_JIS-2004", "Shift_JISX0213", + ]) + def test_unsupported_non_bmp(self, encoding): + parser = expat.ParserCreate() + c = '\U00020e6d\U00028e36' + data = (f'\n' + f'{c}').encode(encoding) + with self.assertRaises(expat.ExpatError): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'UTF-7', + "unicode-escape", "raw-unicode-escape", + "EUC-KR", + "GB18030", + "HZ-GB-2312", + "ISO-2022-JP", "ISO-2022-JP-1", "ISO-2022-JP-2004", + "ISO-2022-JP-2", "ISO-2022-JP-3", "ISO-2022-JP-EXT", + "ISO-2022-KR", + ]) + def test_unsupported_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(ValueError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(ValueError): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'cp037', 'cp273', 'cp424', 'cp500', 'cp864', 'cp875', + 'cp1026', 'cp1140', + 'mac_arabic', 'mac_farsi', + ]) + def test_incompatible_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode(encoding) + with self.assertRaises(expat.ExpatError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaisesRegex(expat.ExpatError, 'unknown encoding'): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'hex_codec', 'rot_13', + ]) + def test_non_text_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'\n' + '').encode() + with self.assertRaises(LookupError): + parser.Parse(data, True) + + def test_undefined_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(UnicodeError): + parser.Parse(data, True) + + def test_unknown_encoding(self): + parser = expat.ParserCreate() + data = b'\n' + with self.assertRaises(LookupError): + parser.Parse(data, True) + + @support.subTests('sample,exception', [ + (b' \xa1', UnicodeDecodeError), # crashed + (b' \xa1 \xa1', expat.ExpatError), + ]) + def test_multibyte_encoding_errors(self, sample, exception): + parser = expat.ParserCreate() + data = b'\n' + sample + with self.assertRaises(exception): + parser.Parse(data, True) + class NamespaceSeparatorTest(unittest.TestCase): def test_legal(self): # Tests that make sure we get errors when the namespace_separator value @@ -712,6 +873,20 @@ def test_change_size_2(self): parser.Parse(xml2, True) self.assertEqual(self.n, 4) + @support.requires_resource('cpu') + @support.requires_resource('walltime') + @support.bigmemtest(size=2**31, memuse=4, dry_run=False) + def test_large_character_data_does_not_crash(self, size): + # See https://github.com/python/cpython/issues/148441 + parser = expat.ParserCreate() + parser.buffer_text = True + parser.buffer_size = 2**31 - 1 # INT_MAX + N = 2049 * (1 << 20) - 3 # Character data greater than INT_MAX + self.assertGreater(N, parser.buffer_size) + parser.CharacterDataHandler = lambda text: None + xml_data = b"" + b"A" * N + b"" + self.assertEqual(parser.Parse(xml_data, True), 1) + class ElementDeclHandlerTest(unittest.TestCase): def test_trigger_leak(self): # Unfixed, this test would leak the memory of the so-called @@ -735,7 +910,7 @@ def test_trigger_leak(self): def test_deeply_nested_content_model(self): # This should raise a RecursionError and not crash. # See https://github.com/python/cpython/issues/145986. - N = 500_000 + N = 800_000 data = ( b' None: result = completer.get_completions(code) self.assertEqual(result is None, expected is None) if result: - compl, act = result + compl, _values, act = result self.assertEqual(compl, expected[0]) self.assertEqual(act is None, expected[1] is None) if act: @@ -1716,6 +1719,50 @@ def test_suggestions_and_messages(self) -> None: new_imports = sys.modules.keys() - _imported self.assertSetEqual(new_imports, expected_imports) + def test_colorize_import_completions(self) -> None: + theme = get_theme() + type_color = theme.fancycompleter.type + module_color = theme.fancycompleter.module + R = ANSIColors.RESET + + colorize = lambda names, values: colorize_matches(names, values, theme) + config = ReadlineConfig(colorize_completions=colorize) + reader = ReadlineAlikeReader( + console=FakeConsole(events=[]), + config=config, + ) + + # "from collections import de" -> defaultdict (type) and deque (type) + reader.buffer = list("from collections import de") + reader.pos = len(reader.buffer) + names, action = reader.get_module_completions() + self.assertEqual(names, [ + f"{type_color}defaultdict{R}", + f"{type_color}deque{R}", + ]) + self.assertIsNone(action) + + # "from importlib.m" has submodule completions colored as modules + reader.buffer = list("from importlib.m") + reader.pos = len(reader.buffer) + names, action = reader.get_module_completions() + self.assertEqual(names, [ + f"{module_color}importlib.machinery{R}", + f"{module_color}importlib.metadata{R}", + ]) + self.assertIsNone(action) + + # Make sure attributes take precedence over submodules when both exist + # Here we're using `unittest.main` which happens to be both a module and an attribute + reader.buffer = list("from unittest import m") + reader.pos = len(reader.buffer) + names, action = reader.get_module_completions() + self.assertEqual(names, [ + f"{type_color}main{R}", # Ensure that `main` is colored as an attribute (class in this case) + f"{module_color}mock{R}", + ]) + self.assertIsNone(action) + # Audit hook used to check for stdlib modules import side-effects # Defined globally to avoid adding one hook per test run (refleak) diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 3c55b6bdaeee9e8..ebbd06213c69aff 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -125,6 +125,30 @@ def test_gen_colors_keyword_highlighting(self): ("import", "keyword"), ], ), + ( + "case +1", + [ + ("case", "soft_keyword"), + ("+", "op"), + ("1", "number"), + ], + ), + ( + "match +1", + [ + ("match", "soft_keyword"), + ("+", "op"), + ("1", "number"), + ], + ), + ( + "match -1", + [ + ("match", "soft_keyword"), + ("-", "op"), + ("1", "number"), + ], + ), ] for code, expected_highlights in cases: with self.subTest(code=code): diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 5e19307b815a1bc..4ab615b150002c8 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,7 +1,7 @@ from test.support import (gc_collect, bigmemtest, _2G, cpython_only, captured_stdout, check_disallow_instantiation, linked_to_musl, - warnings_helper, SHORT_TIMEOUT, Stopwatch, requires_resource) + SHORT_TIMEOUT, Stopwatch, requires_resource) import locale import re import string @@ -1288,80 +1288,90 @@ def test_not_literal(self): self.assertEqual(re.search(r"\s([^a])", " b").group(1), "b") self.assertEqual(re.search(r"\s([^a]*)", " bb").group(1), "bb") - def test_possible_set_operations(self): + def test_set_operations(self): + # UTS #18 RL1.3 set operations in character classes: '--' (difference), + # '&&' (intersection) and '||' (union) are operators on the matched + # character; '~~' (symmetric difference) is still reserved + # (FutureWarning). s = bytes(range(128)).decode() - with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: - p = re.compile(r'[0-9--1]') - self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list('-./0123456789')) - with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: - self.assertEqual(re.findall(r'[0-9--2]', s), list('-./0123456789')) - self.assertEqual(w.filename, __file__) + # Set difference A--B == A and not B. + self.assertEqual(re.findall(r'[0-9--1]', s), list('023456789')) + self.assertEqual(re.findall(r'[0-9--2]', s), list('013456789')) + self.assertEqual(re.findall(r'[%--1]', s), list('%')) + # A leading '-' is a literal, so this stays a range. self.assertEqual(re.findall(r'[--1]', s), list('-./01')) - - with self.assertWarnsRegex(FutureWarning, 'Possible set difference') as w: - p = re.compile(r'[%--1]') - self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list("%&'()*+,-1")) - - with self.assertWarnsRegex(FutureWarning, 'Possible set difference ') as w: - p = re.compile(r'[%--]') - self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list("%&'()*+,-")) - - with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: - p = re.compile(r'[0-9&&1]') - self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list('&0123456789')) - with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: - self.assertEqual(re.findall(r'[0-8&&1]', s), list('&012345678')) - self.assertEqual(w.filename, __file__) - - with self.assertWarnsRegex(FutureWarning, 'Possible set intersection ') as w: - p = re.compile(r'[\d&&1]') - self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list('&0123456789')) - + # A dangling operator (empty operand) is an error. + self.assertRaises(re.PatternError, re.compile, r'[%--]') + + # Set intersection A&&B == A and B. + self.assertEqual(re.findall(r'[0-9&&1]', s), list('1')) + self.assertEqual(re.findall(r'[0-8&&1]', s), list('1')) + self.assertEqual(re.findall(r'[\d&&1]', s), list('1')) + # A leading '&' is a literal. self.assertEqual(re.findall(r'[&&1]', s), list('&1')) - with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: - p = re.compile(r'[0-9||a]') - self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list('0123456789a|')) - - with self.assertWarnsRegex(FutureWarning, 'Possible set union ') as w: - p = re.compile(r'[\d||a]') + # Nested sets and lookbehind-mapped operands. + self.assertEqual(re.findall(r'[a-z--[aeiou]]', s), + list('bcdfghjklmnpqrstvwxyz')) + self.assertEqual(re.findall(r'[\w&&[a-z]]', s), + list('abcdefghijklmnopqrstuvwxyz')) + # Operators chain and mix left-to-right. + self.assertEqual(re.findall(r'[a-z--[aeiou]--[xyz]]', s), + list('bcdfghjklmnpqrstvw')) + self.assertEqual(re.findall(r'[\w&&[a-z]&&[m-z]]', s), + list('mnopqrstuvwxyz')) + # A negated set operation: [^A--B] == complement of (A minus B). + self.assertEqual(re.findall(r'[^a-z--aeiou]', s), + [c for c in s if not ('a' <= c <= 'z' and c not in 'aeiou')]) + # A nested operand may be complemented or itself a set operation; it is + # used directly as the assertion body. + self.assertEqual(re.findall(r'[a-z--[^m]]', s), list('m')) + self.assertEqual(re.findall(r'[\w&&[a-c--b]]', s), list('ac')) + self.assertEqual(re.findall(r'[a-f&&[^bc]]', s), list('adef')) + # A nested set is the whole operand; it cannot be mixed with loose + # members (write the members in the set instead). + self.assertEqual(re.findall(r'[a-c--[ab]]', s), list('c')) + self.assertRaises(re.PatternError, re.compile, r'[a-c--[ab]d]') + self.assertRaises(re.PatternError, re.compile, r'[a-c--[ab][c]]') + # A '[' is a nested set only immediately after a set operator; + # elsewhere it is a literal, so these stay backward compatible. + self.assertEqual(re.findall(r'[*?[]', s), list('*?[')) + self.assertEqual(re.findall(r'[a[b]', s), list('[ab')) + self.assertEqual(re.findall(r'[^[]', 'a[b'), list('ab')) + # A '[' at the start of a class also stays a literal (the position is + # reserved, so it still warns) and keeps its historical meaning. + with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: + p = re.compile(r'[[a-z]]') self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list('0123456789a|')) - + self.assertEqual(p.findall('a]b[c'), ['a]']) # {[, a-z} then a literal ']' + with self.assertWarnsRegex(FutureWarning, 'Possible nested set '): + re.compile(r'[[:digit:]]') + # A nested set after an operator does not warn. + with warnings.catch_warnings(): + warnings.simplefilter('error', FutureWarning) + re.compile(r'[a-z--[aeiou]]') + + # Set union A||B == A or B (an explicit form of [AB]); flat operands + # merge into one charset, otherwise the operations are alternated. + self.assertEqual(re.findall(r'[0-9||a]', s), list('0123456789a')) + self.assertEqual(re.findall(r'[\d||a]', s), list('0123456789a')) + self.assertEqual(re.findall(r'[a-z--m||0-9]', s), + list('0123456789abcdefghijklnopqrstuvwxyz')) + # A leading '|' is a literal. self.assertEqual(re.findall(r'[||1]', s), list('1|')) + # '~~' remains reserved. + with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: p = re.compile(r'[0-9~~1]') self.assertEqual(w.filename, __file__) self.assertEqual(p.findall(s), list('0123456789~')) - with self.assertWarnsRegex(FutureWarning, 'Possible set symmetric difference ') as w: - p = re.compile(r'[\d~~1]') + self.assertEqual(re.findall(r'[\d~~1]', s), list('0123456789~')) self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list('0123456789~')) - self.assertEqual(re.findall(r'[~~1]', s), list('1~')) - with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: - p = re.compile(r'[[0-9]|]') - self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list('0123456789[]')) - with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: - self.assertEqual(re.findall(r'[[0-8]|]', s), list('012345678[]')) - self.assertEqual(w.filename, __file__) - - with self.assertWarnsRegex(FutureWarning, 'Possible nested set ') as w: - p = re.compile(r'[[:digit:]|]') - self.assertEqual(w.filename, __file__) - self.assertEqual(p.findall(s), list(':[]dgit')) - def test_search_coverage(self): self.assertEqual(re.search(r"\s(b)", " b").group(1), "b") self.assertEqual(re.search(r"a\s", "a ").group(0), "a ") @@ -1780,11 +1790,10 @@ def test_bug_6561(self): for x in not_decimal_digits: self.assertIsNone(re.match(r'^\d$', x)) - @warnings_helper.ignore_warnings(category=DeprecationWarning) # gh-80480 array('u') def test_empty_array(self): # SF buf 1647541 import array - for typecode in 'bBhuwHiIlLfd': + for typecode in array.typecodes: a = array.array(typecode) self.assertIsNone(re.compile(b"bla").match(a)) self.assertEqual(re.compile(b"").match(a).groups(), ()) @@ -2816,6 +2825,12 @@ def test_atomic_group(self): 17: SUCCESS ''') + def test_debug_charset_bitmap(self): + # gh-152100: disassembling a charset that compiles to a CHARSET/ + # BIGCHARSET bitmap must not fail (the disassembler needs _CODEBITS). + out = get_debug_out(r'[aeiou]') + self.assertIn('CHARSET', out) + def test_possesive_repeat_one(self): self.assertEqual(get_debug_out(r'a?+'), '''\ POSSESSIVE_REPEAT 0 1 @@ -2941,7 +2956,7 @@ def test_immutable(self): tp.foo = 1 def test_overlap_table(self): - f = re._compiler._generate_overlap_table + f = re._optimizer._generate_overlap_table self.assertEqual(f(""), []) self.assertEqual(f("a"), [0]) self.assertEqual(f("abcd"), [0, 0, 0, 0]) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 02f6e0c74b5ce84..6d30d267cd5ad29 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -41,7 +41,11 @@ ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) -LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' +LOG_PREFIX = ( + r'[0-9]+:[0-9]+:[0-9]+ ' + r'(?:load avg: [0-9]+\.[0-9]{2} )?' + r'(?:mem: [0-9]+\.[0-9] (?:MiB|GiB) )?' +) RESULT_REGEX = ( 'passed', 'failed', @@ -2283,7 +2287,8 @@ def test_python_opts(self): proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True) + text=True, + env=support.make_clean_env()) self.assertEqual(proc.returncode, 0, proc) def test_add_python_opts(self): diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index ba736b0806eb1b0..6ea27c463f31484 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -45,8 +45,8 @@ def test_fsize_ismax(self): resource.setrlimit(resource.RLIMIT_FSIZE, (max, max)) resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - @unittest.skipIf(sys.platform == "vxworks", - "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipIf(sys.platform in ("vxworks", "cygwin"), + f"setting RLIMIT_FSIZE is not supported on {sys.platform}") @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_enforced(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) @@ -84,8 +84,8 @@ def test_fsize_too_big(self): except (OverflowError, ValueError): pass - @unittest.skipIf(sys.platform == "vxworks", - "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipIf(sys.platform in ("vxworks", "cygwin"), + f"setting RLIMIT_FSIZE is not supported on {sys.platform}") @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_not_too_big(self): (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) @@ -163,8 +163,8 @@ def test_getrusage(self): pass # Issue 6083: Reference counting bug - @unittest.skipIf(sys.platform == "vxworks", - "setting RLIMIT_CPU is not supported on VxWorks") + @unittest.skipIf(sys.platform in ("vxworks", "cygwin"), + f"setting RLIMIT_CPU is not supported on {sys.platform}") @unittest.skipUnless(hasattr(resource, 'RLIMIT_CPU'), 'requires resource.RLIMIT_CPU') def test_setrusage_refcount(self): limits = resource.getrlimit(resource.RLIMIT_CPU) diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index a8914953ce9eb48..c0b5a4da8cb2569 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch import builtins +import types import rlcompleter from test.support import MISSING_C_DOCSTRINGS @@ -106,19 +107,35 @@ def test_excessive_getattr(self): # we use __dir__ and __getattr__ in class Foo to create a "magic" # class attribute 'bar'. This forces `getattr` to call __getattr__ # (which is doesn't necessarily do). - class Foo: + # Test 1: Attribute returns None + class FooReturnsNone: calls = 0 - bar = '' + bar = None def __getattribute__(self, name): if name == 'bar': self.calls += 1 return None return super().__getattribute__(name) - f = Foo() - completer = rlcompleter.Completer(dict(f=f)) - self.assertEqual(completer.complete('f.b', 0), 'f.bar') - self.assertEqual(f.calls, 1) + f1 = FooReturnsNone() + completer1 = rlcompleter.Completer(dict(f=f1)) + self.assertEqual(completer1.complete('f.b', 0), 'f.bar') + self.assertEqual(f1.calls, 1) + + # Test 2: Attribute returns non-None value + class FooReturnsValue: + calls = 0 + bar = '' + def __getattribute__(self, name): + if name == 'bar': + self.calls += 1 + return '' + return super().__getattribute__(name) + + f2 = FooReturnsValue() + completer2 = rlcompleter.Completer(dict(f=f2)) + self.assertEqual(completer2.complete('f.b', 0), 'f.bar') + self.assertEqual(f2.calls, 1) def test_property_method_not_called(self): class Foo: @@ -135,6 +152,57 @@ def bar(self): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertFalse(f.property_called) + def test_released_memoryview_completion_works(self): + mv = memoryview(b"abc") + mv.release() + + self.assertIsInstance(type(mv).shape, types.GetSetDescriptorType) + self.assertIsInstance(type(mv).strides, types.GetSetDescriptorType) + + completer = rlcompleter.Completer(dict(mv=mv)) + matches = completer.attr_matches('mv.') + + # These are getset descriptors on memoryview and should be completed + # without evaluating the released-memoryview getters. + self.assertIn('mv.shape', matches) + self.assertIn('mv.strides', matches) + + def test_member_descriptor_not_evaluated(self): + class Foo: + __slots__ = ("boom",) + boom_accesses = 0 + + def __getattribute__(self, name): + if name == "boom": + type(self).boom_accesses += 1 + raise RuntimeError("boom access should be skipped") + return super().__getattribute__(name) + + self.assertIsInstance(Foo.boom, types.MemberDescriptorType) + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom_accesses, 0) + + def test_raising_descriptor_completion_works(self): + class ExplodingDescriptor: + def __init__(self): + self.instance_get_calls = 0 + + def __get__(self, obj, owner): + if obj is None: + return self + self.instance_get_calls += 1 + raise RuntimeError("descriptor getter exploded") + + class Foo: + boom = ExplodingDescriptor() + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom.instance_get_calls, 0) def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when @@ -144,6 +212,7 @@ class Foo: completer = rlcompleter.Completer(dict(f=Foo())) self.assertEqual(completer.complete('f.', 0), 'f.bar') + @unittest.mock.patch('rlcompleter._readline_available', False) def test_complete(self): completer = rlcompleter.Completer() diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index e33723cc70c877b..cd1477037e94b74 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -15,14 +15,18 @@ class BaseRobotTest: good = [] bad = [] site_maps = None + expected_output = None def __init_subclass__(cls): super().__init_subclass__() # Remove tests that do nothing. - if not cls.good: - cls.test_good_urls = None - if not cls.bad: - cls.test_bad_urls = None + if issubclass(cls, unittest.TestCase): + if not cls.good: + cls.test_good_urls = None + if not cls.bad: + cls.test_bad_urls = None + if cls.expected_output is None: + cls.test_string_formatting = None def setUp(self): lines = io.StringIO(self.robots_txt).readlines() @@ -50,6 +54,8 @@ def test_bad_urls(self): def test_site_maps(self): self.assertEqual(self.parser.site_maps(), self.site_maps) + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -61,6 +67,56 @@ class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): good = ['/', '/test.html'] bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] +class SimpleExampleTest(BaseRobotTest, unittest.TestCase): + # Example from RFC 9309, section 5.1. + robots_txt = """\ +User-Agent: * +Disallow: *.gif$ +Disallow: /example/ +Allow: /publications/ + +User-Agent: foobot +Disallow:/ +Allow:/example/page.html +Allow:/example/allowed.gif + +User-Agent: barbot +User-Agent: bazbot +Disallow: /example/page.html + +User-Agent: quxbot + """ + good = [ + '/', '/publications/', + ('foobot', '/example/page.html'), ('foobot', '/example/allowed.gif'), + ('barbot', '/'), ('barbot', '/example/'), + ('barbot', '/example/allowed.gif'), + ('barbot', '/example/disallowed.gif'), + ('barbot', '/publications/'), + ('barbot', '/publications/allowed.gif'), + ('bazbot', '/'), ('bazbot', '/example/'), + ('bazbot', '/example/allowed.gif'), + ('bazbot', '/example/disallowed.gif'), + ('bazbot', '/publications/'), + ('bazbot', '/publications/allowed.gif'), + ('quxbot', '/'), ('quxbot', '/example/'), + ('quxbot', '/example/page.html'), ('quxbot', '/example/allowed.gif'), + ('quxbot', '/example/disallowed.gif'), + ('quxbot', '/publications/'), + ('quxbot', '/publications/allowed.gif'), + ] + bad = [ + '/example/', '/example/page.html', '/example/allowed.gif', + '/example/disallowed.gif', + '/publications/allowed.gif', + ('foobot', '/'), ('foobot', '/example/'), + ('foobot', '/example/disallowed.gif'), + ('foobot', '/publications/'), + ('foobot', '/publications/allowed.gif'), + ('barbot', '/example/page.html'), + ('bazbot', '/example/page.html'), + ] + class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -102,7 +158,7 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): User-agent: * Disallow: / """ - good = [] + good = ['/robots.txt'] bad = ['/cyberworld/map/index.html', '/', '/tmp/'] @@ -137,6 +193,7 @@ def test_request_rate(self): class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): robots_txt = '' good = ['/foo'] + expected_output = '' class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): @@ -203,35 +260,209 @@ class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase): - # the order of User-agent should be correct. note - # that this file is incorrect because "Googlebot" is a - # substring of "Googlebot-Mobile" + # the order of User-agent should not matter robots_txt = """\ User-agent: Googlebot Disallow: / +Allow: /folder1/ User-agent: Googlebot-Mobile Allow: / +Disallow: /folder1/ """ agent = 'Googlebot' bad = ['/something.jpg'] + good = ['/folder1/myfile.html'] class UserAgentGoogleMobileTest(UserAgentOrderingTest): - agent = 'Googlebot-Mobile' + agent = 'Googlebot-mobile' + bad = ['/folder1/myfile.html'] + good = ['/something.jpg'] -class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase): - # Google also got the order wrong. You need - # to specify the URLs from more specific to more general +class LongestMatchTest(BaseRobotTest, unittest.TestCase): + # Based on example from RFC 9309, section 5.2. robots_txt = """\ -User-agent: Googlebot -Allow: /folder1/myfile.html -Disallow: /folder1/ +User-agent: * +Allow: /example/page/ +Disallow: /example/page/disallowed.gif +Allow: /example/ """ - agent = 'googlebot' - good = ['/folder1/myfile.html'] - bad = ['/folder1/anotherfile.html'] + good = ['/example/', '/example/page/'] + bad = ['/example/page/disallowed.gif'] + + +class LongestMatchWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Allow: /example/page/ +Disallow: *.gif +Allow: /example/ + """ + good = ['/example/', '/example/page/'] + bad = ['/example/page/disallowed.gif', '/x.gif'] + + +class AllowWinsEqualMatchTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: /spam +Disallow: /spam + """ + good = ['/spam', '/spam/'] + + +class AllowWinsEqualFullMatchTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: /spam$ +Disallow: /spam +Disallow: /eggs$ +Allow: /eggs +Disallow: /eggs$ + """ + good = ['/spam', '/eggs', '/eggs/'] + bad = ['/spam/'] + + +class AllowWinsEqualMatchWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: *am +Disallow: /spam +Disallow: *gs +Allow: /eggs +Disallow: *gs + """ + good = ['/spam', '/eggs', '/spam/', '/eggs/'] + + +class MergeGroupsTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path + +User-agent: spambot +Disallow: /another/path + """ + agent = 'spambot' + bad = ['/some/path', '/another/path'] + + +class UserAgentStartsGroupTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path +User-agent: eggsbot +Disallow: /another/path + """ + good = [('spambot', '/'), ('spambot', '/another/path'), + ('eggsbot', '/'), ('eggsbot', '/some/path')] + bad = [('spambot', '/some/path'), ('eggsbot', '/another/path')] + expected_output = """\ +User-agent: spambot +Disallow: /some/path + +User-agent: eggsbot +Disallow: /another/path\ +""" + +class IgnoreEmptyLinesTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot + +User-agent: eggsbot +Disallow: /some/path + +Disallow: /another/path + """ + good = [('spambot', '/'), ('eggsbot', '/')] + bad = [ + ('spambot', '/some/path'), ('spambot', '/another/path'), + ('eggsbot', '/some/path'), ('eggsbot', '/another/path'), + ] + expected_output = """\ +User-agent: spambot +User-agent: eggsbot +Disallow: /some/path +Disallow: /another/path\ +""" + + +class IgnoreRulesWithoutUserAgentTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +Disallow: /some/path + +User-agent: * +Disallow: /another/path + """ + good = ['/', '/some/path'] + bad = ['/another/path'] + expected_output = """\ +User-agent: * +Disallow: /another/path\ +""" + + +class EmptyGroupTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /some/path + +User-agent: spambot + """ + agent = 'spambot' + good = ['/', '/some/path'] + expected_output = """\ +User-agent: * +Disallow: /some/path + +User-agent: spambot +Allow:\ +""" + + +class WeirdPathTest(BaseRobotTest, unittest.TestCase): + robots_txt = f"""\ +User-agent: * +Disallow: /a$$$ +Disallow: /b$z +Disallow: /c*** +Disallow: /d***z +Disallow: /e*$**$$ +Disallow: /f*$**$$z +Disallow: /g$*$$** +Disallow: /h$*$$**z + """ + good = ['/ax', '/a$$', '/b', '/bz', '/b$z', '/d', '/f', '/fz', + '/f$$$z', '/fx$y$$z', '/gx', '/g$$$', '/g$x$$y', '/h', '/hz', + '/h$$$z', '/h$x$$yz'] + bad = ['/a', '/c', '/cxy', '/dz', '/dxyz', '/dxzy', '/e', '/exy', + '/e$$', '/ex$y$', '/g'] + expected_output = """\ +User-agent: * +Disallow: /a$ +Disallow: /c* +Disallow: /d*z +Disallow: /e*$ +Disallow: /g$\ +""" + + +class PathWithManyWildcardsTest(BaseRobotTest, unittest.TestCase): + # This test would take many years if use naive translation to regular + # expression (* -> .*). + N = 50 + robots_txt = f"""\ +User-agent: * +Disallow: /{'*a'*N}*b + """ + good = ['/' + 'a'*N + 'a'] + bad = ['/' + 'a'*N + 'b'] class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): @@ -245,25 +476,13 @@ class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): good = ['/some/path', '/some/path?', '/some/path%3Fname=value', '/some/path?name%3Dvalue', '/another/path', '/another/path%3F', - '/yet/one/path?name=value%26more'] + '/yet/one/path?name=value%26more', + '/some/pathxname=value'] bad = ['/some/path?name=value' '/another/path?', '/another/path?name=value', '/yet/one/path?name=value&more'] -class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): - # obey first * entry (#4108) - robots_txt = """\ -User-agent: * -Disallow: /some/path - -User-agent: * -Disallow: /another/path - """ - good = ['/another/path'] - bad = ['/some/path'] - - class PercentEncodingTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * @@ -365,17 +584,60 @@ class StringFormattingTest(BaseRobotTest, unittest.TestCase): """ expected_output = """\ -User-agent: cybermapper -Disallow: /some/path - User-agent: * Crawl-delay: 1 Request-rate: 3/15 -Disallow: /cyberworld/map/\ +Disallow: /cyberworld/map/ + +User-agent: cybermapper +Disallow: /some/path\ """ - def test_string_formatting(self): - self.assertEqual(str(self.parser), self.expected_output) + +class ConstructedStringFormattingTest(unittest.TestCase): + def test_empty(self): + parser = urllib.robotparser.RobotFileParser() + self.assertEqual(str(parser), '') + + def test_group_without_rules(self): + parser = urllib.robotparser.RobotFileParser() + entry = urllib.robotparser.Entry() + entry.useragents = ['spambot'] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['hambot'] + entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['eggsbot'] + parser._add_entry(entry) + self.assertEqual(str(parser), """\ +User-agent: spambot +Allow: + +User-agent: hambot +Disallow: /ham + +User-agent: eggsbot +Allow:\ +""") + + def test_group_without_user_agent(self): + parser = urllib.robotparser.RobotFileParser() + entry = urllib.robotparser.Entry() + entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['spambot'] + entry.rulelines = [urllib.robotparser.RuleLine('/spam', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.rulelines = [urllib.robotparser.RuleLine('/eggs', False)] + parser._add_entry(entry) + self.assertEqual(str(parser), """\ +User-agent: spambot +Disallow: /spam\ +""") @unittest.skipUnless( @@ -384,26 +646,23 @@ def test_string_formatting(self): ) class BaseLocalNetworkTestCase: - def setUp(self): + @classmethod + def setUpClass(cls): # clear _opener global variable - self.addCleanup(urllib.request.urlcleanup) + cls.addClassCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) + cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler) + cls.addClassCleanup(cls.server.server_close) - self.t = threading.Thread( + t = threading.Thread( name='HTTPServer serving', - target=self.server.serve_forever, + target=cls.server.serve_forever, # Short poll interval to make the test finish quickly. # Time between requests is short enough that we won't wake # up spuriously too many times. kwargs={'poll_interval':0.01}) - self.t.daemon = True # In case this function raises. - self.t.start() - - def tearDown(self): - self.server.shutdown() - self.t.join() - self.server.server_close() + cls.enterClassContext(threading_helper.start_threads([t])) + cls.addClassCleanup(cls.server.shutdown) SAMPLE_ROBOTS_TXT = b'''\ @@ -425,7 +684,6 @@ def do_GET(self): def log_message(self, format, *args): pass - @threading_helper.reap_threads def testRead(self): # Test that reading a weird robots.txt doesn't fail. addr = self.server.server_address @@ -440,31 +698,79 @@ def testRead(self): self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) - self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) -class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): +class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): - self.send_error(403, "Forbidden access") + self.send_error(self.server.return_code) def log_message(self, format, *args): pass - @threading_helper.reap_threads - def testPasswordProtectedSite(self): + def setUp(self): + # Make sure that a valid code is set in the test. + self.server.return_code = None + + def testUnauthorized(self): + self.server.return_code = 401 addr = self.server.server_address - url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testForbidden(self): + self.server.return_code = 403 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testNotFound(self): + self.server.return_code = 404 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/path/file.html')) + + def testTeapot(self): + self.server.return_code = 418 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream')) + + def testServiceUnavailable(self): + self.server.return_code = 503 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' robots_url = url + "/robots.txt" parser = urllib.robotparser.RobotFileParser() parser.set_url(url) parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) @support.requires_working_socket() @@ -476,6 +782,7 @@ class NetworkTestCase(unittest.TestCase): @classmethod def setUpClass(cls): support.requires('network') + cls.addClassCleanup(urllib.request.urlcleanup) with socket_helper.transient_internet(cls.base_url): cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) cls.parser.read() @@ -495,7 +802,7 @@ def test_basic(self): def test_can_fetch(self): self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere'))) self.assertFalse(self.parser.can_fetch('Nutch', self.base_url)) - self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian'))) + self.assertTrue(self.parser.can_fetch('Nutch', self.url('brian'))) self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats'))) self.assertFalse(self.parser.can_fetch('*', self.url('webstats'))) self.assertTrue(self.parser.can_fetch('*', self.base_url)) diff --git a/Lib/test/test_samply_profiler.py b/Lib/test/test_samply_profiler.py index ec0ed37ffd047b7..f9ab9207c3c23d8 100644 --- a/Lib/test/test_samply_profiler.py +++ b/Lib/test/test_samply_profiler.py @@ -240,5 +240,29 @@ def compile_trampolines_for_all_functions(): self.assertIn(line, child_perf_file_contents) +@unittest.skipUnless(samply_command_works(), "samply command doesn't work") +class TestSamplyProfilerWithJitDump(unittest.TestCase, TestSamplyProfilerMixin): + # Regression test for gh-150723: exercises the binary jitdump backend + # (-Xperf_jit) end to end through samply, unlike TestSamplyProfiler which + # uses the textual perf-map backend (-Xperf). + def run_samply(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_samply(script_dir, sys.executable, "-Xperf_jit", script) + return run_samply(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.jit_files = set(pathlib.Path("/tmp/").glob("jit-*.dump")) + self.jit_files |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = set(pathlib.Path("/tmp/").glob("jit-*.dump")) + files_to_delete |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + files_to_delete -= self.jit_files + for file in files_to_delete: + file.unlink() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 2a355abdeeb30fb..2adaee81b063085 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -342,6 +342,14 @@ def testQuote(self): self.assertRaises(TypeError, shlex.quote, 42) self.assertRaises(TypeError, shlex.quote, b"abc") + def testForceQuote(self): + self.assertEqual(shlex.quote("spam"), "spam") + self.assertEqual(shlex.quote("spam", force=False), "spam") + self.assertEqual(shlex.quote("spam", force=True), "'spam'") + self.assertEqual(shlex.quote("spam eggs", force=False), "'spam eggs'") + self.assertEqual(shlex.quote("spam eggs", force=True), "'spam eggs'") + self.assertEqual(shlex.quote("two's-complement", force=False), "'two'\"'\"'s-complement'") + def testJoin(self): for split_command, command in [ (['a ', 'b'], "'a ' b"), diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 13a3487382dfcfb..bb901220fb408c1 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2914,6 +2914,23 @@ def test_destinsrc_false_positive(self): finally: os_helper.rmtree(TESTFN) + @os_helper.skip_unless_symlink + def test_destinsrc_symlink_bypass(self): + tmp = self.mkdtemp() + src = os.path.join(tmp, 'src') + os.makedirs(src) + # tmp/link -> tmp (one level up) + link = os.path.join(tmp, 'link') + os.symlink(tmp, link) + # raw path: tmp/link/src/sub - no src prefix in string space + # real path: tmp/src/sub - physically inside src + dst = os.path.join(link, 'src', 'sub') + self.assertTrue( + shutil._destinsrc(src, dst), + msg='_destinsrc failed to detect dst inside src via symlink ' + '(dst=%s, src=%s)' % (dst, src), + ) + @os_helper.skip_unless_symlink @mock_rename def test_move_file_symlink(self): @@ -3580,8 +3597,6 @@ def test_module_all_attribute(self): if hasattr(os, 'statvfs') or os.name == 'nt': target_api.append('disk_usage') self.assertEqual(set(shutil.__all__), set(target_api)) - with self.assertWarns(DeprecationWarning): - from shutil import ExecError # noqa: F401 if __name__ == '__main__': diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index d6cc22558ec4faf..2cad18a69ff7c02 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -833,6 +833,8 @@ def test_itimer_real(self): # Issue 3864, unknown if this affects earlier versions of freebsd also @unittest.skipIf(sys.platform in ('netbsd5',) or is_apple_mobile, 'itimer not reliable (does not mix well with threading) on some BSDs.') + @unittest.skipIf(sys.platform == 'cygwin', + "Cygwin doesn't support ITIMER_VIRTUAL") def test_itimer_virtual(self): self.itimer = signal.ITIMER_VIRTUAL signal.signal(signal.SIGVTALRM, self.sig_vtalrm) @@ -850,6 +852,8 @@ def test_itimer_virtual(self): # and the handler should have been called self.assertEqual(self.hndl_called, True) + @unittest.skipIf(sys.platform == 'cygwin', + "Cygwin doesn't support ITIMER_PROF") def test_itimer_prof(self): self.itimer = signal.ITIMER_PROF signal.signal(signal.SIGPROF, self.sig_prof) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index ac69e2cbdbbe547..bcb51ed7d8476a8 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -187,19 +187,17 @@ def test_addsitedir(self): self.pth_file_tests(pth_file) def test_addsitedir_explicit_flush(self): - # addsitedir() reads .pth files and, with - # defer_processing_start_files=True, accumulates pending state - # without flushing. A subsequent process_startup_files() call - # then applies the paths and runs the import lines. + # StartupState.addsitedir() reads .pth files and accumulates pending + # state without flushing. A subsequent state.process() call then + # applies the paths and runs the import lines. pth_file = PthFile() # Ensure we have a clean slate. pth_file.cleanup(prep=True) with pth_file.create(): - # Pass defer_processing_start_files=True to prevent flushing. - site.addsitedir(pth_file.base_dir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(pth_file.base_dir) self.assertNotIn(pth_file.imported, sys.modules) - site.process_startup_files() + state.process() self.pth_file_tests(pth_file) def test_addsitedir_dotfile(self): @@ -235,6 +233,20 @@ def test_addsitedir_hidden_file_attribute(self): self.assertNotIn(site.makepath(pth_file.good_dir_path)[0], sys.path) self.assertIn(pth_file.base_dir, sys.path) + def test_sitedir_variable(self): + # gh-149671: setuptools use of `-nspkg.pth` files in Python < 3.15. + code = '; '.join(( + # Code used by "-nspkg.pth" files generated by setuptools. + "import sys", + "sitedir = sys._getframe(1).f_locals['sitedir']", + "print(sitedir)", + )) + pth_dir, pth_fn = self.make_pth(code) + with support.captured_stdout() as stdout: + known_paths = site.addpackage(pth_dir, pth_fn, set()) + sitedir = stdout.getvalue().rstrip() + self.assertEqual(sitedir, pth_dir) + # This tests _getuserbase, hence the double underline # to distinguish from a test for getuserbase def test__getuserbase(self): @@ -423,15 +435,14 @@ def create(self): Used as a context manager: self.cleanup() is called on exit. """ - FILE = open(self.file_path, 'w') - try: - print("#import @bad module name", file=FILE) - print("\n", file=FILE) - print("import %s" % self.imported, file=FILE) - print(self.good_dirname, file=FILE) - print(self.bad_dirname, file=FILE) - finally: - FILE.close() + with open(self.file_path, 'w') as fp: + print(f"""\ +#import @bad module name +import {self.imported} +{self.good_dirname} +{self.bad_dirname} +""", file=fp) + os.mkdir(self.good_dir_path) try: yield self @@ -456,6 +467,7 @@ def cleanup(self, prep=False): if os.path.exists(self.bad_dir_path): os.rmdir(self.bad_dir_path) + class ImportSideEffectTests(unittest.TestCase): """Test side-effects from importing 'site'.""" @@ -545,7 +557,6 @@ def test_customization_modules_on_startup(self): output = subprocess.check_output([sys.executable, '-s', '-c', '""']) self.assertNotIn(eyecatcher, output.decode('utf-8')) - @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') @@ -915,68 +926,121 @@ class StartFileTests(unittest.TestCase): def setUp(self): self.enterContext(import_helper.DirsOnSysPath()) self.tmpdir = self.sitedir = self.enterContext(os_helper.temp_dir()) - # Save and clear all pending dicts. - self.saved_entrypoints = site._pending_entrypoints.copy() - self.saved_syspaths = site._pending_syspaths.copy() - self.saved_importexecs = site._pending_importexecs.copy() - site._pending_entrypoints.clear() - site._pending_syspaths.clear() - site._pending_importexecs.clear() + # Each test gets its own StartupState to batch the parsing and + # explicitly invoke the processing. Seed with an empty known_paths + # so dedup is not influenced by the current sys.path. + self.state = site.StartupState(known_paths=set()) - def tearDown(self): - site._pending_entrypoints = self.saved_entrypoints.copy() - site._pending_syspaths = self.saved_syspaths.copy() - site._pending_importexecs = self.saved_importexecs.copy() + def _make_start(self, content, name='testpkg', basedir=None): + """Write a .start file and return its basename. - def _make_start(self, content, name='testpkg'): - """Write a .start file and return its basename.""" + ``basedir`` defaults to ``self.tmpdir``. Pass an explicit directory + when the .start file needs to live somewhere other than the test's + primary tmpdir (e.g. a nested user-site). + """ basename = f"{name}.start" - filepath = os.path.join(self.tmpdir, basename) + filepath = os.path.join(self.tmpdir if basedir is None else basedir, basename) with open(filepath, 'w', encoding='utf-8') as f: f.write(content) return basename - def _make_pth(self, content, name='testpkg'): - """Write a .pth file and return its basename.""" + def _make_pth(self, content, name='testpkg', basedir=None): + """Write a .pth file and return its basename. + + ``basedir`` defaults to ``self.tmpdir``. Pass an explicit directory + when the .pth file needs to live somewhere other than the test's + primary tmpdir (e.g. a nested user-site). + """ basename = f"{name}.pth" - filepath = os.path.join(self.tmpdir, basename) + filepath = os.path.join(self.tmpdir if basedir is None else basedir, basename) with open(filepath, 'w', encoding='utf-8') as f: f.write(content) return basename - def _all_entrypoints(self): - """Flatten _pending_entrypoints dict into a list of (filename, entry) tuples.""" + def _make_mod(self, contents, name='mod', *, package=False, on_path=False): + """Write an importable module (or package), returning its parent dir.""" + extdir = os.path.join(self.sitedir, 'extdir') + os.makedirs(extdir, exist_ok=True) + + # Put the code in a package's dunder-init or flat module. + if package: + pkgdir = os.path.join(extdir, name) + os.mkdir(pkgdir) + modpath = os.path.join(pkgdir, '__init__.py') + else: + modpath = os.path.join(extdir, f'{name}.py') + + with open(modpath, 'w') as fp: + fp.write(contents) + + self.addCleanup(sys.modules.pop, name, None) + if on_path: + # Don't worry, DirsOnSysPath() in setUp() will clean this up. + sys.path.insert(0, extdir) + return extdir + + def _all_entrypoints(self, state=None): + """Flatten state._entrypoints into a list of (filename, entry) tuples.""" result = [] - for filename, entries in site._pending_entrypoints.items(): + state = self.state if state is None else state + for filename, entries in state._entrypoints.items(): for entry in entries: result.append((filename, entry)) return result - def _just_entrypoints(self): - return [entry for filename, entry in self._all_entrypoints()] + def _just_entrypoints(self, state=None): + return [entry for filename, entry in self._all_entrypoints(state)] + + # There are two classes of tests here. Tests that start with `test_impl_` + # know details about the implementation and they access non-public methods + # and data structures to perform focused functional tests. + # + # Tests that start with `test_addsitedir_` are end-to-end tests that ensure + # integration semantics and functionality as a caller of the public + # surfaces would see. + + # --- Basic StartupState implementation tests --- + + def test_impl_startupstate_defaults_to_sys_path(self): + sys.path.insert(0, self.sitedir) + state = site.StartupState() + self.assertIn(site.makepath(self.sitedir)[1], state._known_paths) + + def test_impl_startupstate_uses_supplied_known_paths(self): + known_paths = set() + state = site.StartupState(known_paths) + state.addsitedir(self.sitedir) + self.assertIs(state._known_paths, known_paths) + self.assertIn(site.makepath(self.sitedir)[1], known_paths) - # --- _read_start_file tests --- + # --- StartupState._read_start_file tests --- - def test_read_start_file_basic(self): + def test_impl_read_start_file_basic(self): self._make_start("os.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_multiple_entries(self): + def test_impl_read_start_file_multiple_entries(self): self._make_start("os.path:join\nos.path:exists\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:exists']) + self.assertEqual( + self.state._entrypoints[fullname], + ['os.path:join', 'os.path:exists'], + ) - def test_read_start_file_comments_and_blanks(self): + def test_impl_read_start_file_comments_and_blanks(self): self._make_start("# a comment\n\nos.path:join\n \n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], ['os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_accepts_all_non_blank_lines(self): + def test_impl_read_start_file_accepts_all_non_blank_lines(self): # Syntax validation is deferred to entry-point execution time # (where pkgutil.resolve_name(strict=True) enforces the strict # pkg.mod:callable form), so parsing accepts every non-blank, @@ -989,9 +1053,9 @@ def test_read_start_file_accepts_all_non_blank_lines(self): "os.path:join\n" # valid ) self._make_start(content, name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], [ + self.assertEqual(self.state._entrypoints[fullname], [ 'os.path', 'pkg.mod:', ':callable', @@ -999,155 +1063,167 @@ def test_read_start_file_accepts_all_non_blank_lines(self): 'os.path:join', ]) - def test_read_start_file_empty(self): + def test_impl_read_start_file_empty(self): # PEP 829: an empty .start file is still registered as present - # (with an empty entry-point list) so that it suppresses `import` + # (with an empty entry point list) so that it suppresses `import` # lines in any matching .pth file. self._make_start("", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) + self.assertEqual(self.state._entrypoints, {fullname: []}) - def test_read_start_file_comments_only(self): + def test_impl_read_start_file_comments_only(self): # As with an empty file, a comments-only .start file is registered # as present so it can suppress matching .pth `import` lines. self._make_start("# just a comment\n# another\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {fullname: []}) + self.assertEqual(self.state._entrypoints, {fullname: []}) - def test_read_start_file_nonexistent(self): + def test_impl_read_start_file_nonexistent(self): with captured_stderr(): - site._read_start_file(self.tmpdir, 'nonexistent.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state._read_start_file(self.tmpdir, 'nonexistent.start') + self.assertEqual(self.state._entrypoints, {}) @unittest.skipUnless(hasattr(os, 'chflags'), 'test needs os.chflags()') - def test_read_start_file_hidden_flags(self): + def test_impl_read_start_file_hidden_flags(self): self._make_start("os.path:join\n", name='foo') filepath = os.path.join(self.tmpdir, 'foo.start') st = os.stat(filepath) os.chflags(filepath, st.st_flags | stat.UF_HIDDEN) - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state._read_start_file(self.sitedir, 'foo.start') + self.assertEqual(self.state._entrypoints, {}) - def test_read_start_file_duplicates_not_deduplicated(self): + def test_impl_one_start_file_with_duplicates_not_deduplicated(self): # PEP 829: duplicate entry points are NOT deduplicated. self._make_start("os.path:join\nos.path:join\n", name='foo') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints[fullname], - ['os.path:join', 'os.path:join']) + self.assertEqual( + self.state._entrypoints[fullname], + ['os.path:join', 'os.path:join'], + ) + + def test_impl_two_start_files_with_duplicates_not_deduplicated(self): + self._make_start("os.path:join", name="foo") + self._make_start("os.path:join", name="bar") + self.state._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'bar.start') + self.assertEqual( + self._just_entrypoints(), + ['os.path:join', 'os.path:join'], + ) - def test_read_start_file_accepts_utf8_bom(self): + def test_impl_read_start_file_accepts_utf8_bom(self): # PEP 829: .start files MUST be utf-8-sig (UTF-8 with optional BOM). filepath = os.path.join(self.tmpdir, 'foo.start') with open(filepath, 'wb') as f: f.write(b'\xef\xbb\xbf' + b'os.path:join\n') - site._read_start_file(self.sitedir, 'foo.start') + self.state._read_start_file(self.sitedir, 'foo.start') fullname = os.path.join(self.sitedir, 'foo.start') self.assertEqual( - site._pending_entrypoints[fullname], ['os.path:join']) + self.state._entrypoints[fullname], ['os.path:join'] + ) - def test_read_start_file_invalid_utf8_silently_skipped(self): - # PEP 829: .start files MUST be utf-8-sig. Unlike .pth, there is - # no locale-encoding fallback -- a .start file that is not valid + def test_impl_read_start_file_invalid_utf8_silently_skipped(self): + # PEP 829: .start files MUST be utf-8-sig. Unlike .pth files, there + # is no locale-encoding fallback. A .start file that is not valid # UTF-8 is silently skipped, with no key registered in - # _pending_entrypoints and no output to stderr (parsing errors - # are reported only under -v). + # state._entrypoints and no output to stderr (parsing errors are + # reported only under -v). filepath = os.path.join(self.tmpdir, 'foo.start') with open(filepath, 'wb') as f: # Bare continuation byte -- invalid as a UTF-8 start byte. f.write(b'\x80\x80\x80\n') with captured_stderr() as err: - site._read_start_file(self.sitedir, 'foo.start') - self.assertEqual(site._pending_entrypoints, {}) + self.state._read_start_file(self.sitedir, 'foo.start') + self.assertEqual(self.state._entrypoints, {}) self.assertEqual(err.getvalue(), "") - def test_two_start_files_with_duplicates_not_deduplicated(self): - self._make_start("os.path:join", name="foo") - self._make_start("os.path:join", name="bar") - site._read_start_file(self.sitedir, 'foo.start') - site._read_start_file(self.sitedir, 'bar.start') - self.assertEqual(self._just_entrypoints(), - ['os.path:join', 'os.path:join']) - - # --- _read_pth_file tests --- + # --- StartupState._read_pth_file tests --- - def test_read_pth_file_paths(self): + def test_impl_read_pth_file_paths(self): subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) self._make_pth("mylib\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state._read_pth_file(self.sitedir, 'foo.pth') fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths[fullname]) + self.assertIn((fullname, subdir), self.state._path_entries) - def test_read_pth_file_imports_collected(self): + def test_impl_read_pth_file_imports_collected(self): self._make_pth("import sys\n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state._read_pth_file(self.sitedir, 'foo.pth') fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertEqual(site._pending_importexecs[fullname], ['import sys']) + self.assertEqual( + self.state._importexecs[fullname], ['import sys'] + ) - def test_read_pth_file_comments_and_blanks(self): + def test_impl_read_pth_file_comments_and_blanks(self): self._make_pth("# comment\n\n \n", name='foo') - site._read_pth_file(self.sitedir, 'foo.pth', set()) - self.assertEqual(site._pending_syspaths, {}) - self.assertEqual(site._pending_importexecs, {}) + self.state._read_pth_file(self.sitedir, 'foo.pth') + self.assertEqual(self.state._path_entries, []) + self.assertEqual(self.state._importexecs, {}) - def test_read_pth_file_deduplication(self): + def test_impl_read_pth_file_deduplication(self): subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) - known_paths = set() + # self.state._known_paths acts as the deduplication ledger across + # both reads. self._make_pth("mylib\n", name='a') self._make_pth("mylib\n", name='b') - site._read_pth_file(self.sitedir, 'a.pth', known_paths) - site._read_pth_file(self.sitedir, 'b.pth', known_paths) - # Only one entry across both files. - all_dirs = [] - for dirs in site._pending_syspaths.values(): - all_dirs.extend(dirs) + self.state._read_pth_file(self.sitedir, 'a.pth') + self.state._read_pth_file(self.sitedir, 'b.pth') + # There is only one entry across both files. + all_dirs = [dir_ for filename, dir_ in self.state._path_entries] self.assertEqual(all_dirs, [subdir]) - def test_read_pth_file_bad_line_continues(self): - # PEP 829: errors on individual lines don't abort the file. + def test_impl_read_pth_file_bad_line_continues(self): + # PEP 829: errors on individual lines don't abort processing the file. subdir = os.path.join(self.sitedir, 'goodpath') os.mkdir(subdir) self._make_pth("abc\x00def\ngoodpath\n", name='foo') with captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) + self.state._read_pth_file(self.sitedir, 'foo.pth') fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn((fullname, subdir), self.state._path_entries) def _flags_with_verbose(self, verbose): # Build a sys.flags clone with verbose overridden but every # other field preserved, so unrelated reads like # sys.flags.optimize during io.open_code() continue to work. - attrs = {name: getattr(sys.flags, name) - for name in sys.flags.__match_args__} + attrs = { + name: getattr(sys.flags, name) + for name in sys.flags.__match_args__ + } attrs['verbose'] = verbose return SimpleNamespace(**attrs) - def test_read_pth_file_parse_error_silent_by_default(self): + def test_impl_read_pth_file_parse_error_silent_by_default(self): # PEP 829: parse-time errors are silent unless -v is given. - # Force the error path by making makepath() raise. + # Force the error path by making makepath() raise an exception. self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(False)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('site.makepath', side_effect=ValueError("boom")), + mock.patch('sys.flags', self._flags_with_verbose(False)), + captured_stderr() as err, + ): + self.state._read_pth_file(self.sitedir, 'foo.pth') self.assertEqual(err.getvalue(), "") - def test_read_pth_file_parse_error_reported_under_verbose(self): + def test_impl_read_pth_file_parse_error_reported_under_verbose(self): # PEP 829: parse-time errors are reported when -v is given. self._make_pth("badline\n", name='foo') - with mock.patch('site.makepath', side_effect=ValueError("boom")), \ - mock.patch('sys.flags', self._flags_with_verbose(True)), \ - captured_stderr() as err: - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('site.makepath', side_effect=ValueError("boom")), + mock.patch('sys.flags', self._flags_with_verbose(True)), + captured_stderr() as err, + ): + self.state._read_pth_file(self.sitedir, 'foo.pth') out = err.getvalue() self.assertIn('Error in', out) self.assertIn('foo.pth', out) - def test_read_pth_file_locale_fallback(self): + def test_impl_read_pth_file_locale_fallback(self): # PEP 829: .pth files that fail UTF-8 decoding fall back to the # locale encoding for backward compatibility (deprecated in # 3.15, to be removed in 3.20). Mock locale.getencoding() so @@ -1158,186 +1234,215 @@ def test_read_pth_file_locale_fallback(self): # \xe9 is invalid UTF-8 but valid in latin-1. with open(filepath, 'wb') as f: f.write(b'# caf\xe9 comment\nmylib\n') - with mock.patch('locale.getencoding', return_value='latin-1'), \ - captured_stderr(): - site._read_pth_file(self.sitedir, 'foo.pth', set()) + with ( + mock.patch('locale.getencoding', return_value='latin-1'), + captured_stderr(), + ): + self.state._read_pth_file(self.sitedir, 'foo.pth') fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn((fullname, subdir), self.state._path_entries) - # --- _execute_start_entrypoints tests --- + # --- StartupState._execute_start_entrypoints tests --- - def test_execute_entrypoints_with_callable(self): - # Entrypoint with callable is invoked. - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + def test_impl_execute_entrypoints_with_callable(self): + # An entry point with a callable. + self._make_mod("""\ called = False def startup(): global called called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) +""", name='epmod', package=True, on_path=True) fullname = os.path.join(self.sitedir, 'epmod.start') - site._pending_entrypoints[fullname] = ['epmod:startup'] - site._execute_start_entrypoints() + self.state._entrypoints[fullname] = ['epmod:startup'] + self.state._execute_start_entrypoints() import epmod self.assertTrue(epmod.called) - def test_execute_entrypoints_import_error(self): - # Import error prints traceback but continues. + def test_impl_execute_entrypoints_import_error(self): + # Import errors print a traceback and continue. fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ - 'nosuchmodule_xyz:func', 'os.path:join'] + self.state._entrypoints[fullname] = [ + 'nosuchmodule_xyz:func', 'os.path:join', + ] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() self.assertIn('nosuchmodule_xyz', err.getvalue()) # os.path:join should still have been called (no exception for it) - def test_execute_entrypoints_strict_syntax_rejection(self): - # PEP 829: only the strict pkg.mod:callable form is valid. - # At entry-point execution, pkgutil.resolve_name(strict=True) - # raises ValueError for invalid syntax; the invalid entry is - # reported and execution continues with the next one. + def test_impl_execute_entrypoints_strict_syntax_rejection(self): + # PEP 829: only the strict pkg.mod:callable form is valid. At entry + # point execution time, pkgutil.resolve_name(strict=True) raises a + # ValueError for the invalid syntax. The invalid entry is reported + # and execution continues with the next one. fullname = os.path.join(self.sitedir, 'bad.start') - site._pending_entrypoints[fullname] = [ + self.state._entrypoints[fullname] = [ 'os.path', # no colon 'pkg.mod:', # empty callable ':callable', # empty module 'pkg.mod:callable:extra', # multiple colons ] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() out = err.getvalue() self.assertIn('Invalid entry point syntax', out) - for bad in ('os.path', 'pkg.mod:', ':callable', - 'pkg.mod:callable:extra'): + for bad in ( + 'os.path', + 'pkg.mod:', + ':callable', + 'pkg.mod:callable:extra', + ): self.assertIn(bad, out) - def test_execute_entrypoints_callable_error(self): - # Callable that raises prints traceback but continues. - mod_dir = os.path.join(self.sitedir, 'badmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + def test_impl_execute_entrypoints_callable_error(self): + # A callable that errors prints a traceback but continues. + self._make_mod("""\ def fail(): raise RuntimeError("boom") -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'badmod', None) +""", name='badmod', package=True, on_path=True) fullname = os.path.join(self.sitedir, 'badmod.start') - site._pending_entrypoints[fullname] = ['badmod:fail'] + self.state._entrypoints[fullname] = ['badmod:fail'] with captured_stderr() as err: - site._execute_start_entrypoints() + self.state._execute_start_entrypoints() self.assertIn('RuntimeError', err.getvalue()) self.assertIn('boom', err.getvalue()) - def test_execute_entrypoints_duplicates_called_twice(self): + def test_impl_execute_entrypoints_duplicates_called_twice(self): # PEP 829: duplicate entry points execute multiple times. - mod_dir = os.path.join(self.sitedir, 'countmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + self._make_mod("""\ call_count = 0 def bump(): global call_count call_count += 1 -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'countmod', None) +""", name='countmod', package=False, on_path=True) fullname = os.path.join(self.sitedir, 'countmod.start') - site._pending_entrypoints[fullname] = [ - 'countmod:bump', 'countmod:bump'] - site._execute_start_entrypoints() + self.state._entrypoints[fullname] = [ + 'countmod:bump', 'countmod:bump', + ] + self.state._execute_start_entrypoints() import countmod self.assertEqual(countmod.call_count, 2) - # --- _exec_imports tests --- + # --- StartupState._exec_imports tests --- - def test_exec_imports_suppressed_by_matching_start(self): + def test_impl_exec_imports_suppressed_by_matching_start(self): # Import lines from foo.pth are suppressed when foo.start exists. - pth_fullname = os.path.join(self.sitedir, 'foo.pth') - start_fullname = os.path.join(self.sitedir, 'foo.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should not exec the import line; no error expected. - site._exec_imports() + self._make_mod("""\ +call_count = 0 +def bump(incr=2): + global call_count + call_count += incr +""", name='countmod', package=False, on_path=True) + self._make_start("countmod:bump\n", name='foo') + self._make_pth("import countmod; countmod.bump(1)\n", name='foo') + self.state._read_pth_file(self.sitedir, 'foo.pth') + self.state._read_start_file(self.sitedir, 'foo.start') + self.state._exec_imports() + self.state._execute_start_entrypoints() + import countmod + # This will be 2 because the entry point is called with no + # arguments, and the .pth import line is never exec'd. + self.assertEqual(countmod.call_count, 2) - def test_exec_imports_not_suppressed_by_different_start(self): + def test_impl_exec_imports_not_suppressed_by_different_start(self): # Import lines from foo.pth are NOT suppressed by bar.start. + self._make_mod("""\ +call_count = 0 +def bump(): + global call_count + call_count += 1 +""", name='countmod', package=False, on_path=True) pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'bar.start') - site._pending_importexecs[pth_fullname] = ['import sys'] - site._pending_entrypoints[start_fullname] = ['os.path:join'] - # Should execute the import line without error. - site._exec_imports() + self.state._importexecs[pth_fullname] = ['import countmod; countmod.bump()'] + self.state._entrypoints[start_fullname] = ['os.path:join'] + self.state._exec_imports() + import countmod + self.assertEqual(countmod.call_count, 1) - def test_exec_imports_suppressed_by_empty_matching_start(self): + def test_impl_exec_imports_suppressed_by_empty_matching_start(self): self._make_start("", name='foo') self._make_pth("import epmod; epmod.startup()", name='foo') - mod_dir = os.path.join(self.sitedir, 'epmod') - os.mkdir(mod_dir) - init_file = os.path.join(mod_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write("""\ + self._make_mod("""\ called = False def startup(): global called called = True -""") - sys.path.insert(0, self.sitedir) - self.addCleanup(sys.modules.pop, 'epmod', None) - site._read_pth_file(self.sitedir, 'foo.pth', set()) - site._read_start_file(self.sitedir, 'foo.start') - site._exec_imports() +""", name='epmod', package=True, on_path=True) + self.state._read_pth_file(self.sitedir, 'foo.pth') + self.state._read_start_file(self.sitedir, 'foo.start') + self.state._exec_imports() import epmod self.assertFalse(epmod.called) - # --- _extend_syspath tests --- + # --- StartupState._extend_syspath tests --- - def test_extend_syspath_existing_dir(self): + def test_impl_extend_syspath_existing_dir(self): subdir = os.path.join(self.sitedir, 'extlib') os.mkdir(subdir) - site._pending_syspaths['test.pth'] = [subdir] - site._extend_syspath() + self.state._path_entries.append(('test.pth', subdir)) + self.state._extend_syspath() self.assertIn(subdir, sys.path) - def test_extend_syspath_nonexistent_dir(self): - nosuch = os.path.join(self.sitedir, 'nosuchdir') - site._pending_syspaths['test.pth'] = [nosuch] + def test_impl_extend_syspath_nonexistent_dir(self): + nonesuch = os.path.join(self.sitedir, 'nosuchdir') + self.state._path_entries.append(('test.pth', nonesuch)) with captured_stderr() as err: - site._extend_syspath() - self.assertNotIn(nosuch, sys.path) + self.state._extend_syspath() + self.assertNotIn(nonesuch, sys.path) self.assertIn('does not exist', err.getvalue()) # --- addsitedir integration tests --- + def test_addsitedir_pth_import_skipped_when_matching_start_exists(self): + # PEP 829: an empty .start file disables the matching .pth's import + # lines, even when the .start has no entry points of its own. + self._make_mod("flag = False\n", name='suppressed', on_path=True) + self._make_start("", name='foo') + self._make_pth( + "import suppressed; suppressed.flag = True\n", + name='foo') + site.addsitedir(self.sitedir, set()) + import suppressed + self.assertFalse( + suppressed.flag, + "import line in foo.pth should be suppressed by foo.start") + + def test_addsitedir_dotfile_start_entrypoint_not_executed(self): + # .start files starting with '.' are skipped, so their entry + # points must not run. + self._make_mod("""\ +called = False +def hook(): + global called + called = True +""", + name='dotted', on_path=True) + self._make_start("dotted:hook\n", name='.hidden') + site.addsitedir(self.sitedir, set()) + import dotted + self.assertFalse(dotted.called) + + def test_addsitedir_dedups_paths_across_pth_files(self): + # PEP 829: when multiple .pth files reference the same path within + # a single addsitedir() invocation, the path is appended to + # sys.path exactly once. + subdir = os.path.join(self.sitedir, 'shared') + os.mkdir(subdir) + self._make_pth("shared\n", name='a') + self._make_pth("shared\n", name='b') + before = sys.path.count(subdir) + site.addsitedir(self.sitedir, set()) + self.assertEqual(sys.path.count(subdir), before + 1) + def test_addsitedir_discovers_start_files(self): # addsitedir() should discover .start files and accumulate entries. self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn('os.path:join', site._pending_entrypoints[fullname]) - - def test_addsitedir_start_suppresses_pth_imports(self): - # When foo.start exists, import lines in foo.pth are skipped - # at flush time by _exec_imports(). - self._make_start("os.path:join\n", name='foo') - self._make_pth("import sys\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - pth_fullname = os.path.join(self.sitedir, 'foo.pth') - start_fullname = os.path.join(self.sitedir, 'foo.start') - # Import line was collected... - self.assertIn('import sys', - site._pending_importexecs.get(pth_fullname, [])) - # ...but _exec_imports() will skip it because foo.start exists. - site._exec_imports() + self.assertIn( + 'os.path:join', state._entrypoints[fullname] + ) def test_addsitedir_pth_paths_still_work_with_start(self): # Path lines in .pth files still work even when a .start file exists. @@ -1345,74 +1450,129 @@ def test_addsitedir_pth_paths_still_work_with_start(self): os.mkdir(subdir) self._make_start("os.path:join\n", name='foo') self._make_pth("mylib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn((fullname, subdir), state._path_entries) def test_addsitedir_start_alphabetical_order(self): # Multiple .start files are discovered alphabetically. self._make_start("os.path:join\n", name='zzz') self._make_start("os.path:exists\n", name='aaa') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - all_entries = self._all_entrypoints() - entries = [entry for _, entry in all_entries] + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) + entries = self._just_entrypoints(state) idx_a = entries.index('os.path:exists') idx_z = entries.index('os.path:join') self.assertLess(idx_a, idx_z) - def test_addsitedir_pth_before_start(self): - # PEP 829: .pth files are scanned before .start files. - # Create a .pth and .start with the same basename; verify - # the .pth data is collected before .start data. + def test_addsitedir_pth_and_start(self): + # Create a .pth and .start with the same basename; verify both the + # .pth data and .start data is collected. subdir = os.path.join(self.sitedir, 'mylib') os.mkdir(subdir) self._make_pth("mylib\n", name='foo') self._make_start("os.path:join\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) # Both should be collected. pth_fullname = os.path.join(self.sitedir, 'foo.pth') start_fullname = os.path.join(self.sitedir, 'foo.start') - self.assertIn(subdir, site._pending_syspaths.get(pth_fullname, [])) - self.assertIn('os.path:join', - site._pending_entrypoints.get(start_fullname, [])) - - def test_addsitedir_dotfile_start_ignored(self): - # .start files starting with '.' are skipped. Defer flushing so - # the assertion against _pending_entrypoints is meaningful; - # otherwise process_startup_files() would clear the dict - # regardless of whether the dotfile was picked up. + self.assertIn((pth_fullname, subdir), state._path_entries) + self.assertIn( + 'os.path:join', + state._entrypoints.get(start_fullname, []), + ) + + def test_impl_addsitedir_skips_dotfile_start(self): + # .start files starting with '.' are skipped. + # This will create `.hidden.start`. self._make_start("os.path:join\n", name='.hidden') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) - self.assertEqual(site._pending_entrypoints, {}) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) + self.assertEqual(state._entrypoints, {}) def test_addsitedir_standalone_flushes(self): - # When called with known_paths=None (standalone), addsitedir - # flushes immediately so the caller sees the effect. + # Standalone addsitedir creates a per-call StartupState and processes + # it before returning, so the caller sees the effect immediately. subdir = os.path.join(self.sitedir, 'flushlib') os.mkdir(subdir) self._make_pth("flushlib\n", name='foo') - site.addsitedir(self.sitedir) # known_paths=None + # No arguments means state is implied and processing is eager. + site.addsitedir(self.sitedir) self.assertIn(subdir, sys.path) - # Pending dicts should be cleared after flush. - self.assertEqual(site._pending_syspaths, {}) - def test_addsitedir_defer_does_not_flush(self): - # With defer_processing_start_files=True, addsitedir accumulates - # pending state but does not flush; sys.path is updated only when - # process_startup_files() is called explicitly. + def test_addsitedir_explicit_startup_state_does_not_flush(self): + # With an explicit StartupState, addsitedir accumulates pending state + # but does not flush it; sys.path is updated only when process() is + # called explicitly. subdir = os.path.join(self.sitedir, 'acclib') os.mkdir(subdir) self._make_pth("acclib\n", name='foo') - site.addsitedir(self.sitedir, set(), - defer_processing_start_files=True) + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) # Path is pending, not yet on sys.path. self.assertNotIn(subdir, sys.path) fullname = os.path.join(self.sitedir, 'foo.pth') - self.assertIn(subdir, site._pending_syspaths.get(fullname, [])) + self.assertIn((fullname, subdir), state._path_entries) + + def test_addsitedir_startup_state_preserves_site_relative_order(self): + # As pointed out by @ncoghlan in + # https://github.com/python/cpython/issues/150228#issuecomment-4528614952 + # a subtle ordering change was inadvertently introduced where the + # interspersing of the sitedirs with the sys.path extensions they defined + # was lost during batch mode. You'd see all the sitedirs, then all path + # extensions. This test ensures that the old interspersing behavior + # has been restored. + # + # Let's start by creating two sitedirs, each with an extension directory + # which will be added to sys.path by .pth files in the respective sitedirs. + sitedir2 = self.enterContext(os_helper.temp_dir()) + extdir1 = os.path.join(self.sitedir, 'ext1') + extdir2 = os.path.join(sitedir2, 'ext2') + os.mkdir(extdir1) + os.mkdir(extdir2) + self._make_pth(extdir1 + "\n", name='one') + self._make_pth(extdir2 + "\n", name='two', basedir=sitedir2) + # Now create an explicit batch, add each sitedir, then process the + # entire batch. + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) + state.addsitedir(sitedir2) + state.process() + # Ensure that on sys.path we see this interspersed order: + # [sitedir1, extdir1, sitedir2, extdir2] + indexes = [ + sys.path.index(path) + for path in (self.sitedir, extdir1, sitedir2, extdir2) + ] + # If the index ordering is the same, we preserved the intersperse. + self.assertEqual(indexes, sorted(indexes)) + + def test_addsitedir_startup_state_paths_before_entrypoints(self): + # Ensure that sys.path extensions are available by the time + # .start file entry points are called. + extdir = self._make_mod("""\ +called = False +def hook(): + global called + called = True +""") + self.assertNotIn(extdir, sys.path) + self._make_pth("extdir\n", name='extlib') + self._make_start("mod:hook\n", name='extlib') + # Before the startup state is explicitly processed, neither + # the path extension is added, nor the entry point called. + state = site.StartupState(known_paths=set()) + state.addsitedir(self.sitedir) + self.assertNotIn(extdir, sys.path) + self.assertNotIn('mod', sys.modules) + # After processing the batch, sys.path is extended and + # the entry point was called. + state.process() + self.assertIn(extdir, sys.path) + import mod + self.assertTrue(mod.called) def test_pth_path_is_available_to_start_entrypoint(self): # Core PEP 829 invariant: all .pth path extensions are applied to @@ -1420,18 +1580,12 @@ def test_pth_path_is_available_to_start_entrypoint(self): # point may live in a module reachable only via a .pth-extended # path. If the flush phases were inverted, resolving the entry # point would fail with ModuleNotFoundError. - extdir = os.path.join(self.sitedir, 'extdir') - os.mkdir(extdir) - modpath = os.path.join(extdir, 'mod.py') - with open(modpath, 'w') as f: - f.write("""\ + extdir = self._make_mod("""\ called = False def hook(): global called called = True """) - self.addCleanup(sys.modules.pop, 'mod', None) - # extdir is not on sys.path; only the .pth file makes it so. self.assertNotIn(extdir, sys.path) self._make_pth("extdir\n", name='extlib') @@ -1447,6 +1601,156 @@ def hook(): "entry point did not run; .pth path was likely not applied " "before .start entry-point execution") + # --- bugs --- + + # gh-75723 + def test_addsitdir_idempotent_pth(self): + # Adding the same sitedir twice with a known_paths, should not + # process .pth files twice. + extdir = self._make_mod("""\ +_pth_count = 0 +""") + self._make_pth(f"""\ +{extdir} +import mod; mod._pth_count += 1 +""") + dirs = set() + dirs = site.addsitedir(self.sitedir, dirs) + dirs = site.addsitedir(self.sitedir, dirs) + import mod + self.assertEqual(mod._pth_count, 1) + + def test_addsitdir_idempotent_start(self): + # Adding the same sitedir twice with a known_paths, should not + # process .pth files twice. + extdir = self._make_mod("""\ +_pth_count = 0 +def increment(): + global _pth_count + _pth_count += 1 +""") + self._make_pth(f"""\ +{extdir} +""") + self._make_start("""\ +mod:increment +""") + dirs = set() + dirs = site.addsitedir(self.sitedir, dirs) + dirs = site.addsitedir(self.sitedir, dirs) + import mod + self.assertEqual(mod._pth_count, 1) + + # gh-149504 + def test_reentrant_addsitedir_pth(self): + # An import line in a .pth file that calls site.addsitedir() + # must not crash or re-execute outer entries while the outer + # call is still processing its pending startup state. + overlay = self.enterContext(os_helper.temp_dir()) + overlay_pth = os.path.join(overlay, 'overlay.pth') + pkgdir = self.enterContext(os_helper.temp_dir()) + with open(overlay_pth, 'w', encoding='utf-8') as fp: + print(pkgdir, file=fp) + self._make_pth(f"import site; site.addsitedir({overlay!r})\n") + site.addsitedir(self.sitedir, set()) + self.assertIn(overlay, sys.path) + self.assertIn(pkgdir, sys.path) + + # gh-149504 + def test_reentrant_addsitedir_start(self): + # As above, but the re-entry happens from a .start entry point + # instead of a .pth import line. The entry point execution + # phase is vulnerable to the same class of bug. + overlay = self.enterContext(os_helper.temp_dir()) + overlay_pth = os.path.join(overlay, 'overlay.pth') + pkgdir = self.enterContext(os_helper.temp_dir()) + with open(overlay_pth, 'w', encoding='utf-8') as fp: + print(pkgdir, file=fp) + self._make_mod(f"""\ +import site +def bootstrap(): + site.addsitedir({overlay!r}) +""", + name='reenter_helper', on_path=True) + self._make_start("reenter_helper:bootstrap\n") + site.addsitedir(self.sitedir, set()) + self.assertIn(overlay, sys.path) + self.assertIn(pkgdir, sys.path) + + # gh-149819 + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires user-site") + @support.requires_subprocess() + def test_pth_processed_when_sitedir_already_on_path(self): + # A .pth file in a site-packages directory must still be processed by + # site.main() when that directory is already on sys.path at + # interpreter start up, for example in a subprocess that inherits + # PYTHONPATH from its parent. Before the fix, main() seeded + # known_paths with all entries derived from removeduppaths(), and + # addsitedir() then skipped .pth processing for any directory already + # in known_paths. + user_base = self.tmpdir + user_site = site._get_path(user_base) + os.makedirs(user_site) + sentinel = "GH149819_PTH_RAN" + # Writing some text to stderr is the simplest observable side effect. + self._make_pth(f"""\ +import sys; sys.stderr.write({sentinel!r}); sys.stderr.flush() +""", + name='gh149819', + basedir=user_site) + with EnvironmentVarGuard() as env: + # PYTHONUSERBASE points USER_SITE at our temp directory so + # site.main() will call addsitedir() on it, rather than on the + # host interpreter's real user-site. + env['PYTHONUSERBASE'] = user_base + # PYTHONPATH puts that same directory on sys.path before + # site.main() runs in the subprocess. This is what triggers the + # bug: removeduppaths() records it in known_paths, and the unfixed + # addsitedir() then skips .pth processing. + env['PYTHONPATH'] = user_site + result = subprocess.run( + [sys.executable, '-c', ''], + capture_output=True, + check=True, + ) + self.assertIn(sentinel.encode(), result.stderr) + + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires user-site") + @support.requires_subprocess() + def test_start_processed_when_sitedir_already_on_path(self): + # Companion to test_pth_processed_when_sitedir_already_on_path: + # the same dedup-guard skip in addsitedir() suppressed both .pth + # and .start file processing, so verify .start entry points also + # run for a site-packages directory inherited via PYTHONPATH. + user_base = self.tmpdir + user_site = site._get_path(user_base) + os.makedirs(user_site) + sentinel = "GH149819_START_RAN" + # The .start entry point resolves to a callable, so we write a + # tiny importable module that outputs the sentinel text. It lands in + # /extdir. That path is added to PYTHONPATH below so + # the subprocess can import it. + extdir = self._make_mod(f"""\ +import sys +def run(): + sys.stderr.write({sentinel!r}) + sys.stderr.flush() +""", name='gh149819mod') + self._make_start( + 'gh149819mod:run\n', name='gh149819', basedir=user_site + ) + with EnvironmentVarGuard() as env: + # See above for details. + env['PYTHONUSERBASE'] = user_base + env['PYTHONPATH'] = os.pathsep.join([user_site, extdir]) + result = subprocess.run( + [sys.executable, '-c', ''], + capture_output=True, + check=True, + ) + self.assertIn(sentinel.encode(), result.stderr) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 9e03069494345b3..542d97e3f886e24 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -205,6 +205,25 @@ def _have_socket_hyperv(): return True +def _have_udp_lite(): + if not hasattr(socket, "IPPROTO_UDPLITE"): + return False + # Older Android versions block UDPLITE with SELinux. + if support.is_android and platform.android_ver().api_level < 29: + return False + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDPLITE) + except OSError as exc: + # Linux 7.1 removed UDP Lite support + if exc.errno == errno.EPROTONOSUPPORT: + return False + raise + sock.close() + + return True + + @contextlib.contextmanager def socket_setdefaulttimeout(timeout): old_timeout = socket.getdefaulttimeout() @@ -247,10 +266,7 @@ def downgrade_malformed_data_warning(): HAVE_SOCKET_VSOCK = _have_socket_vsock() -# Older Android versions block UDPLITE with SELinux. -HAVE_SOCKET_UDPLITE = ( - hasattr(socket, "IPPROTO_UDPLITE") - and not (support.is_android and platform.android_ver().api_level < 29)) +HAVE_SOCKET_UDPLITE = _have_udp_lite() HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() @@ -1411,6 +1427,7 @@ def testIPv6toString(self): except OSError as e: if e.winerror == 10022: self.skipTest('IPv6 might not be supported') + raise f = lambda a: inet_pton(AF_INET6, a) assertInvalid = lambda a: self.assertRaises( @@ -1438,7 +1455,7 @@ def testIPv6toString(self): assertInvalid('1:2:3:4:5:6:') assertInvalid('1:2:3:4:5:6:7:8:0') # bpo-29972: inet_pton() doesn't fail on AIX - if not AIX: + if not AIX and sys.platform != 'cygwin': assertInvalid('1:2:3:4:5:6:7:8:') self.assertEqual(b'\x00' * 12 + b'\xfe\x2a\x17\x40', @@ -1501,6 +1518,7 @@ def testStringToIPv6(self): except OSError as e: if e.winerror == 10022: self.skipTest('IPv6 might not be supported') + raise f = lambda a: inet_ntop(AF_INET6, a) assertInvalid = lambda a: self.assertRaises( @@ -1985,7 +2003,8 @@ def test_getfqdn_filter_localhost(self): self.assertEqual(socket.getfqdn(), socket.getfqdn("::")) @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') - @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') + @unittest.skipIf(sys.platform in ('win32', 'cygwin'), + 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") @support.skip_android_selinux('if_nameindex') @@ -2019,7 +2038,7 @@ def test_getaddrinfo_ipv6_scopeid_numeric(self): self.assertEqual(sockaddr, ('ff02::1de:c0:face:8d', 1234, 0, ifindex)) @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 required for this test.') - @unittest.skipIf(sys.platform == 'win32', 'does not work on Windows') + @unittest.skipIf(sys.platform in ('win32', 'cygwin'), 'does not work on Windows') @unittest.skipIf(AIX, 'Symbolic scope id does not work') @unittest.skipUnless(hasattr(socket, 'if_nameindex'), "test needs socket.if_nameindex()") @support.skip_android_selinux('if_nameindex') @@ -7174,12 +7193,6 @@ def test_aes_cbc(self): @support.requires_linux_version(4, 9) # see gh-73510 def test_aead_aes_gcm(self): - kernel_version = support._get_kernel_version("Linux") - if kernel_version is not None: - if kernel_version >= (6, 16) and kernel_version < (6, 18): - # See https://github.com/python/cpython/issues/139310. - self.skipTest("upstream Linux kernel issue") - key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') @@ -7518,6 +7531,62 @@ def detach(): pass +class ReentrantMutationTests(unittest.TestCase): + """Regression tests for re-entrant mutation in sendmsg/recvmsg_into. + + These tests verify that mutating sequences during argument parsing + via __buffer__ protocol does not cause crashes. + + See: https://github.com/python/cpython/issues/143988 + """ + + @unittest.skipUnless(hasattr(socket.socket, "sendmsg"), + "sendmsg not supported") + def test_sendmsg_reentrant_data_mutation(self): + seq = [] + + class MutBuffer: + def __init__(self): + self.tripped = False + + def __buffer__(self, flags): + if not self.tripped: + self.tripped = True + seq.clear() + return memoryview(b'Hello') + + seq = [MutBuffer(), b'World', b'Test'] + + left, right = socket.socketpair() + with left, right: + left.sendmsg(seq) + self.assertEqual(right.recv(1024), b'HelloWorldTest') + + @unittest.skipUnless(hasattr(socket.socket, "recvmsg_into"), + "recvmsg_into not supported") + def test_recvmsg_into_reentrant_buffer_mutation(self): + seq = [] + buf1 = bytearray(100) + + class MutBuffer: + def __init__(self): + self.tripped = False + + def __buffer__(self, flags): + if not self.tripped: + self.tripped = True + seq.clear() + return memoryview(buf1) + + seq = [MutBuffer(), bytearray(100), bytearray(100)] + + left, right = socket.socketpair() + with left, right: + left.send(b'Hello World!') + right.recvmsg_into(seq) + self.assertEqual(buf1, b'Hello World!'.ljust(100, b'\x00')) + + def setUpModule(): thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 8ac64b3105708f7..53fffe7cfb56d09 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -387,8 +387,7 @@ def test_utf8_non_utf8_third_line_error(self): b'#third\xa4\n' b'raise RuntimeError\n') self.check_script_error(src, - br"'utf-8' codec can't decode byte|" - br"encoding problem: utf8") + br"'utf-8' codec can't decode byte") def test_crlf(self): src = (b'print(ascii("""\r\n"""))\n') @@ -540,6 +539,20 @@ def check_script_error(self, src, expected, lineno=...): line = line.removeprefix('\ufeff') self.assertIn(line.encode(), err) + def test_coding_spec_unknown_encoding(self): + src = (b'# coding: c1252\n' + b'print("Hi!")\n') + self.check_script_error(src, br"unknown encoding: c1252") + + def test_coding_spec_decode_error(self): + src = (b'# coding: shift-jis\n' + b'print("\xc4\x85")\n') + self.check_script_error(src, br"'shift_jis' codec can't decode byte") + + def test_coding_spec_non_text_encoding(self): + src = (b'# coding: hex_codec\n' + b'print("eggs")\n') + self.check_script_error(src, br"'hex_codec' is not a text encoding") if __name__ == "__main__": diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 73b40e82a96811f..5f6cb527955ca17 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -1400,6 +1400,18 @@ def test_blob_set_empty_slice(self): self.blob[0:0] = b"" self.assertEqual(self.blob[:], self.data) + def test_blob_set_empty_slice_wrong_type(self): + with self.assertRaises(TypeError): + self.blob[5:5] = None + + def test_blob_set_empty_slice_wrong_size(self): + with self.assertRaisesRegex(IndexError, "wrong size"): + self.blob[5:5] = b"123" + + def test_blob_set_empty_slice_correct(self): + self.blob[5:5] = b"" + self.assertEqual(self.blob[:], self.data) + def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"12345" actual = self.cx.execute("select b from test").fetchone()[0] diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index 776659e3b161089..a9abeab31936880 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -146,6 +146,16 @@ def test_sqlite_row_index(self): with self.assertRaises(IndexError): row[complex()] # index must be int or string + def test_delete_connection_row_factory(self): + # gh-149738: deleting row_factory should raise an exception + with self.assertRaises(AttributeError): + del self.con.row_factory + + def test_delete_connection_text_factory(self): + # gh-149738: deleting text_factory should raise an exception + with self.assertRaises(AttributeError): + del self.con.text_factory + def test_sqlite_row_index_unicode(self): row = self.con.execute("select 1 as \xff").fetchone() self.assertEqual(row["\xff"], 1) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f1f7a07701de165..40111d410077950 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -59,10 +59,7 @@ CAN_GET_SELECTED_OPENSSL_SIGALG = ssl.OPENSSL_VERSION_INFO >= (3, 5) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') -HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') -requires_keylog = unittest.skipUnless( - HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') -CAN_SET_KEYLOG = HAS_KEYLOG and os.name != "nt" +CAN_SET_KEYLOG = (os.name != "nt") requires_keylog_setter = unittest.skipUnless( CAN_SET_KEYLOG, "cannot set 'keylog_filename' on Windows" @@ -638,6 +635,7 @@ def test_refcycle(self): del ss self.assertEqual(wr(), None) + @unittest.skipIf(sys.platform == 'cygwin', 'test hangs on Cygwin') def test_wrapped_unconnected(self): # Methods on an unconnected SSLSocket propagate the original # OSError raise by the underlying socket object. @@ -1606,6 +1604,59 @@ def dummycallback(sock, servername, ctx, cycle=ctx): gc.collect() self.assertIs(wr(), None) + @unittest.skipUnless(support.Py_GIL_DISABLED, + "test is only useful if the GIL is disabled") + @threading_helper.requires_working_threading() + def test_sni_callback_race(self): + # Replacing sni_callback while handshakes are in-flight must not + # crash (use-after-free on the callback in free-threaded builds). + client_ctx, server_ctx, hostname = testing_context() + + server_ctx.sni_callback = lambda *a: None + done = threading.Event() + + def do_handshakes(): + while not done.is_set(): + c_in = ssl.MemoryBIO() + c_out = ssl.MemoryBIO() + s_in = ssl.MemoryBIO() + s_out = ssl.MemoryBIO() + client = client_ctx.wrap_bio( + c_in, c_out, server_hostname=hostname) + server = server_ctx.wrap_bio(s_in, s_out, server_side=True) + for _ in range(50): + try: + client.do_handshake() + except ssl.SSLWantReadError: + pass + except ssl.SSLError: + break + if c_out.pending: + s_in.write(c_out.read()) + try: + server.do_handshake() + except ssl.SSLWantReadError: + pass + except ssl.SSLError: + break + if s_out.pending: + c_in.write(s_out.read()) + + def toggle_callback(): + while not done.is_set(): + server_ctx.sni_callback = lambda *a: None + server_ctx.sni_callback = None + + workers = max(4, (os.cpu_count() or 4) * 2) + threads = [threading.Thread(target=do_handshakes) + for _ in range(workers)] + threads.append(threading.Thread(target=toggle_callback)) + + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(threads): + done.set() + self.assertIsNone(cm.exc_value) + def test_cert_store_stats(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) self.assertEqual(ctx.cert_store_stats(), @@ -3578,7 +3629,7 @@ def test_wrong_cert_tls13(self): OSError, 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA|' 'closed by the remote host|Connection reset by peer|' - 'Broken pipe' + 'Broken pipe|Software caused connection abort' ): # TLS 1.3 perform client cert exchange after handshake s.write(b'data') @@ -4532,6 +4583,8 @@ def test_client_sigalgs_mismatch(self): ssl.SSLError, # On handshake failures, some systems raise a ConnectionResetError. ConnectionResetError, + # On handshake failures, Cygwin raises ConnectionAbortedError. + ConnectionAbortedError, # On handshake failures, macOS may raise a BrokenPipeError. # See https://github.com/python/cpython/issues/139504. BrokenPipeError, @@ -5397,7 +5450,6 @@ def keylog_lines(self, fname=os_helper.TESTFN): with open(fname) as f: return len(list(f)) - @requires_keylog def test_keylog_defaults(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) @@ -5425,7 +5477,6 @@ def test_keylog_defaults(self): with self.assertRaises(TypeError): ctx.keylog_filename = 1 - @requires_keylog def test_keylog_filename(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) client_context, server_context, hostname = testing_context() @@ -5466,7 +5517,6 @@ def test_keylog_filename(self): client_context.keylog_filename = None server_context.keylog_filename = None - @requires_keylog @unittest.skipIf(sys.flags.ignore_environment, "test is not compatible with ignore_environment") def test_keylog_env(self): @@ -5640,19 +5690,26 @@ def run(self): def non_linux_skip_if_other_okay_error(self, err): if sys.platform in ("linux", "android"): return # Expect the full test setup to always work on Linux. - if (isinstance(err, ConnectionResetError) or + if (isinstance(err, (ConnectionResetError, ConnectionAbortedError)) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or - re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I) or - re.search('record.layer.failure', str(getattr(err, "reason", "")), re.I) + re.search( + # Matches the following error messages: + # '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)' + # '[SSL: RECORD_LAYER_FAILURE] record layer failure (_ssl.c:1109)' + # '[SSL: HTTP_REQUEST] http request (_ssl.c:1143)' + r'wrong.version.number|record.layer.failure|http.request', + str(getattr(err, "reason", "")), + re.IGNORECASE, + ) ): # On Windows the TCP RST leads to a ConnectionResetError # (ECONNRESET) which Linux doesn't appear to surface to userspace. # If wrap_socket() winds up on the "if connected:" path and doing # the actual wrapping... we get an SSLError from OpenSSL. This is # typically WRONG_VERSION_NUMBER. The same happens on iOS, but - # RECORD_LAYER_FAILURE is the error. + # RECORD_LAYER_FAILURE or HTTP_REQUEST is the error. # - # While appropriate, neither is the scenario we're specifically + # While appropriate, these scenarios aren't what we're specifically # trying to test. The way this test is written is known to work on # Linux. We'll skip it anywhere else that it does not present as # doing so. diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 5dae5dfccac5d32..09ee2d53f98f585 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -134,6 +134,10 @@ def test_windows_feature_macros(self): "PyComplex_ImagAsDouble", "PyComplex_RealAsDouble", "PyComplex_Type", + "PyCriticalSection2_Begin", + "PyCriticalSection2_End", + "PyCriticalSection_Begin", + "PyCriticalSection_End", "PyDescr_NewClassMethod", "PyDescr_NewGetSet", "PyDescr_NewMember", @@ -365,12 +369,18 @@ def test_windows_feature_macros(self): "PyImport_ImportModuleNoBlock", "PyImport_ReloadModule", "PyIndex_Check", + "PyInterpreterGuard_Close", + "PyInterpreterGuard_FromCurrent", + "PyInterpreterGuard_FromView", "PyInterpreterState_Clear", "PyInterpreterState_Delete", "PyInterpreterState_Get", "PyInterpreterState_GetDict", "PyInterpreterState_GetID", "PyInterpreterState_New", + "PyInterpreterView_Close", + "PyInterpreterView_FromCurrent", + "PyInterpreterView_FromMain", "PyIter_Check", "PyIter_Next", "PyIter_NextItem", @@ -691,12 +701,15 @@ def test_windows_feature_macros(self): "PyThreadState_Clear", "PyThreadState_Delete", "PyThreadState_DeleteCurrent", + "PyThreadState_Ensure", + "PyThreadState_EnsureFromView", "PyThreadState_Get", "PyThreadState_GetDict", "PyThreadState_GetFrame", "PyThreadState_GetID", "PyThreadState_GetInterpreter", "PyThreadState_New", + "PyThreadState_Release", "PyThreadState_SetAsyncExc", "PyThreadState_Swap", "PyThread_GetInfo", @@ -739,6 +752,7 @@ def test_windows_feature_macros(self): "PyType_Freeze", "PyType_FromMetaclass", "PyType_FromModuleAndSpec", + "PyType_FromSlots", "PyType_FromSpec", "PyType_FromSpecWithBases", "PyType_GenericAlloc", diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 677a87b51b91925..700c5ac304f7179 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -16,7 +16,8 @@ import sys import unittest from test import support -from test.support import import_helper, requires_IEEE_754 +from test.support import (import_helper, requires_IEEE_754, + skip_if_double_rounding, skip_on_newlib) from decimal import Decimal from fractions import Fraction @@ -28,12 +29,6 @@ # === Helper functions and class === -# Test copied from Lib/test/test_math.py -# detect evidence of double-rounding: fsum is not always correctly -# rounded on machines that suffer from double rounding. -x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer -HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) - def sign(x): """Return -1.0 for negatives, including -0.0, otherwise +1.0.""" return math.copysign(1, x) @@ -2796,9 +2791,9 @@ def test_sqrtprod_helper_function_fundamentals(self): self.assertEqual(sign(actual), sign(expected)) @requires_IEEE_754 - @unittest.skipIf(HAVE_DOUBLE_ROUNDING, - "accuracy not guaranteed on machines with double rounding") + @skip_if_double_rounding @support.cpython_only # Allow for a weaker sumprod() implementation + @skip_on_newlib def test_sqrtprod_helper_function_improved_accuracy(self): # Test a known example where accuracy is improved x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 5ac28870455f4d8..e95cc6db170e243 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -19,6 +19,13 @@ else: glibc_ver = None +def skip_cygwin_locale(): + if sys.platform != 'cygwin': + return + loc = locale.getlocale(locale.LC_TIME)[0] + if loc in ('my_MM', 'or_IN'): + raise unittest.SkipTest('test fails on Cygwin') + class getlang_Tests(unittest.TestCase): """Test _getlang""" @@ -509,6 +516,8 @@ def test_bad_timezone(self): 'my_MM', 'or_IN', 'shn_MM', 'az_IR', 'byn_ER', 'wal_ET', 'lzh_TW') def test_date_time_locale(self): + skip_cygwin_locale() + # Test %c directive loc = locale.getlocale(locale.LC_TIME)[0] if glibc_ver and glibc_ver < (2, 31) and loc == 'br_FR': @@ -536,6 +545,8 @@ def test_date_time_locale(self): 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', 'my_MM', 'shn_MM') def test_date_time_locale2(self): + skip_cygwin_locale() + # Test %c directive loc = locale.getlocale(locale.LC_TIME)[0] if sys.platform.startswith('sunos'): @@ -550,6 +561,8 @@ def test_date_time_locale2(self): 'he_IL', 'eu_ES', 'ar_AE', 'az_IR', 'my_MM', 'or_IN', 'shn_MM', 'lzh_TW') def test_date_locale(self): + skip_cygwin_locale() + # Test %x directive now = time.time() self.roundtrip('%x', slice(0, 3), time.localtime(now)) @@ -567,6 +580,8 @@ def test_date_locale(self): @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM', 'lzh_TW') def test_date_locale2(self): + skip_cygwin_locale() + # Test %x directive loc = locale.getlocale(locale.LC_TIME)[0] if sys.platform.startswith(('sunos', 'aix')): @@ -587,6 +602,8 @@ def test_date_locale2(self): 'ti_ET', 'tig_ER', 'wal_ET', 'lzh_TW', 'ar_SA', 'bg_BG') def test_time_locale(self): + skip_cygwin_locale() + # Test %X directive loc = locale.getlocale(locale.LC_TIME)[0] pos = slice(3, 6) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 1a3db527d3d5b83..d066ae85dfc51a6 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -2428,26 +2428,26 @@ def test_run_abort(self): p.wait() self.assertEqual(-p.returncode, signal.SIGABRT) - def test_CalledProcessError_str_signal(self): + def test_CalledProcessError_str(self): + # command string + err = subprocess.CalledProcessError(2, "fake cmd") + self.assertEqual(str(err), "Command 'fake cmd' returned non-zero exit status 2.") + + # command string with a single-quote + err = subprocess.CalledProcessError(2, "fake ' cmd") + self.assertEqual(str(err), 'Command "fake \' cmd" returned non-zero exit status 2.') + + # command list + err = subprocess.CalledProcessError(2, ["fake", "cmd"]) + self.assertEqual(str(err), "Command ['fake', 'cmd'] returned non-zero exit status 2.") + + # signal err = subprocess.CalledProcessError(-int(signal.SIGABRT), "fake cmd") - error_string = str(err) - # We're relying on the repr() of the signal.Signals intenum to provide - # the word signal, the signal name and the numeric value. - self.assertIn("signal", error_string.lower()) - # We're not being specific about the signal name as some signals have - # multiple names and which name is revealed can vary. - self.assertIn("SIG", error_string) - self.assertIn(str(signal.SIGABRT), error_string) - - def test_CalledProcessError_str_unknown_signal(self): - err = subprocess.CalledProcessError(-9876543, "fake cmd") - error_string = str(err) - self.assertIn("unknown signal 9876543.", error_string) + self.assertEqual(str(err), f"Command 'fake cmd' died with {signal.SIGABRT!r}.") - def test_CalledProcessError_str_non_zero(self): - err = subprocess.CalledProcessError(2, "fake cmd") - error_string = str(err) - self.assertIn("non-zero exit status 2.", error_string) + # unknown signal + err = subprocess.CalledProcessError(-9876543, "fake cmd") + self.assertEqual(str(err), "Command 'fake cmd' died with unknown signal 9876543.") def test_preexec(self): # DISCLAIMER: Setting environment variables is *not* a good use @@ -3719,8 +3719,9 @@ def test_startupinfo(self): # Since Python is a console process, it won't be affected # by wShowWindow, but the argument should be silently # ignored - subprocess.call(ZERO_RETURN_CMD, - startupinfo=startupinfo) + rc = subprocess.call(ZERO_RETURN_CMD, + startupinfo=startupinfo) + self.assertEqual(rc, 0) def test_startupinfo_keywords(self): # startupinfo argument @@ -3735,8 +3736,9 @@ def test_startupinfo_keywords(self): # Since Python is a console process, it won't be affected # by wShowWindow, but the argument should be silently # ignored - subprocess.call(ZERO_RETURN_CMD, - startupinfo=startupinfo) + rc = subprocess.call(ZERO_RETURN_CMD, + startupinfo=startupinfo) + self.assertEqual(rc, 0) def test_startupinfo_copy(self): # bpo-34044: Popen must not modify input STARTUPINFO structure @@ -3765,13 +3767,17 @@ def test_startupinfo_copy(self): self.assertEqual(startupinfo.wShowWindow, subprocess.SW_HIDE) self.assertEqual(startupinfo.lpAttributeList, {"handle_list": []}) + # CREATE_NEW_CONSOLE creates a "popup" window. + @support.requires_resource('gui') def test_creationflags(self): # creationflags argument CREATE_NEW_CONSOLE = 16 sys.stderr.write(" a DOS box should flash briefly ...\n") - subprocess.call(sys.executable + - ' -c "import time; time.sleep(0.25)"', - creationflags=CREATE_NEW_CONSOLE) + rc = subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + support.skip_on_low_desktop_heap_memory_subprocess(rc) + self.assertEqual(rc, 0) def test_invalid_args(self): # invalid arguments should raise ValueError @@ -3849,14 +3855,16 @@ def test_close_fds_with_stdio(self): def test_empty_attribute_list(self): startupinfo = subprocess.STARTUPINFO() startupinfo.lpAttributeList = {} - subprocess.call(ZERO_RETURN_CMD, - startupinfo=startupinfo) + rc = subprocess.call(ZERO_RETURN_CMD, + startupinfo=startupinfo) + self.assertEqual(rc, 0) def test_empty_handle_list(self): startupinfo = subprocess.STARTUPINFO() startupinfo.lpAttributeList = {"handle_list": []} - subprocess.call(ZERO_RETURN_CMD, - startupinfo=startupinfo) + rc = subprocess.call(ZERO_RETURN_CMD, + startupinfo=startupinfo) + self.assertEqual(rc, 0) def test_shell_sequence(self): # Run command through the shell (sequence) @@ -3867,6 +3875,8 @@ def test_shell_sequence(self): env=newenv) with p: self.assertIn(b"physalis", p.stdout.read()) + p.communicate() + self.assertEqual(p.returncode, 0) def test_shell_string(self): # Run command through the shell (string) @@ -3877,6 +3887,8 @@ def test_shell_string(self): env=newenv) with p: self.assertIn(b"physalis", p.stdout.read()) + p.communicate() + self.assertEqual(p.returncode, 0) def test_shell_encodings(self): # Run command through the shell (string) @@ -3889,6 +3901,8 @@ def test_shell_encodings(self): encoding=enc) with p: self.assertIn("physalis", p.stdout.read(), enc) + p.communicate() + self.assertEqual(p.returncode, 0) def test_call_string(self): # call() function with string argument on Windows diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index c748243110df9ff..8c03420c4c5e4b1 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -2,8 +2,6 @@ Test the API of the symtable module. """ -import re -import textwrap import symtable import warnings import unittest @@ -364,87 +362,6 @@ def test_name(self): self.assertEqual(self.spam.lookup("x").get_name(), "x") self.assertEqual(self.Mine.get_name(), "Mine") - def test_class_get_methods(self): - deprecation_mess = ( - re.escape('symtable.Class.get_methods() is deprecated ' - 'and will be removed in Python 3.16.') - ) - - with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): - self.assertEqual(self.Mine.get_methods(), ('a_method',)) - - top = symtable.symtable(TEST_COMPLEX_CLASS_CODE, "?", "exec") - this = find_block(top, "ComplexClass") - - with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): - self.assertEqual(this.get_methods(), ( - 'a_method', 'a_method_pep_695', - 'an_async_method', 'an_async_method_pep_695', - 'a_classmethod', 'a_classmethod_pep_695', - 'an_async_classmethod', 'an_async_classmethod_pep_695', - 'a_staticmethod', 'a_staticmethod_pep_695', - 'an_async_staticmethod', 'an_async_staticmethod_pep_695', - 'a_fakemethod', 'a_fakemethod_pep_695', - 'an_async_fakemethod', 'an_async_fakemethod_pep_695', - 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', - 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', - 'glob_assigned_meth', 'glob_assigned_meth_pep_695', - 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', - )) - - # Test generator expressions that are of type TYPE_FUNCTION - # but will not be reported by get_methods() since they are - # not functions per se. - # - # Other kind of comprehensions such as list, set or dict - # expressions do not have the TYPE_FUNCTION type. - - def check_body(body, expected_methods): - indented = textwrap.indent(body, ' ' * 4) - top = symtable.symtable(f"class A:\n{indented}", "?", "exec") - this = find_block(top, "A") - with self.assertWarnsRegex(DeprecationWarning, deprecation_mess): - self.assertEqual(this.get_methods(), expected_methods) - - # statements with 'genexpr' inside it - GENEXPRS = ( - 'x = (x for x in [])', - 'x = (x async for x in [])', - 'type x[genexpr = (x for x in [])] = (x for x in [])', - 'type x[genexpr = (x async for x in [])] = (x async for x in [])', - 'genexpr = (x for x in [])', - 'genexpr = (x async for x in [])', - 'type genexpr[genexpr = (x for x in [])] = (x for x in [])', - 'type genexpr[genexpr = (x async for x in [])] = (x async for x in [])', - ) - - for gen in GENEXPRS: - # test generator expression - with self.subTest(gen=gen): - check_body(gen, ()) - - # test generator expression + variable named 'genexpr' - with self.subTest(gen=gen, isvar=True): - check_body('\n'.join((gen, 'genexpr = 1')), ()) - check_body('\n'.join(('genexpr = 1', gen)), ()) - - for paramlist in ('()', '(x)', '(x, y)', '(z: T)'): - for func in ( - f'def genexpr{paramlist}:pass', - f'async def genexpr{paramlist}:pass', - f'def genexpr[T]{paramlist}:pass', - f'async def genexpr[T]{paramlist}:pass', - ): - with self.subTest(func=func): - # test function named 'genexpr' - check_body(func, ('genexpr',)) - - for gen in GENEXPRS: - with self.subTest(gen=gen, func=func): - # test generator expression + function named 'genexpr' - check_body('\n'.join((gen, func)), ('genexpr',)) - check_body('\n'.join((func, gen)), ('genexpr',)) - def test_filename_correct(self): ### Bug tickler: SyntaxError file name correct whether error raised ### while parsing or building symbol table. diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3002fa528eab17b..f40da0b79aa4790 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1283,16 +1283,6 @@ def check(tracebacklimit, expected): def test_no_duplicates_in_meta_path(self): self.assertEqual(len(sys.meta_path), len(set(sys.meta_path))) - @unittest.skipUnless(hasattr(sys, "_enablelegacywindowsfsencoding"), - 'needs sys._enablelegacywindowsfsencoding()') - def test__enablelegacywindowsfsencoding(self): - code = ('import sys', - 'sys._enablelegacywindowsfsencoding()', - 'print(sys.getfilesystemencoding(), sys.getfilesystemencodeerrors())') - rc, out, err = assert_python_ok('-c', '; '.join(code)) - out = out.decode('ascii', 'replace').rstrip() - self.assertEqual(out, 'mbcs replace') - @support.requires_subprocess() def test_orig_argv(self): code = textwrap.dedent(''' @@ -1359,6 +1349,26 @@ def test_pystats(self): def test_disable_gil_abi(self): self.assertEqual('t' in sys.abiflags, support.Py_GIL_DISABLED) + def test_int_max_str_digits(self): + old_limit = sys.get_int_max_str_digits() + self.assertIsInstance(old_limit, int) + self.assertGreaterEqual(old_limit, 0) + self.addCleanup(sys.set_int_max_str_digits, old_limit) + + sys.set_int_max_str_digits(0) + self.assertEqual(sys.get_int_max_str_digits(), 0) + + sys.set_int_max_str_digits(2_048) + self.assertEqual(sys.get_int_max_str_digits(), 2_048) + + with self.assertRaises(ValueError): + # the minimum is 640 digits + sys.set_int_max_str_digits(5) + with self.assertRaises(ValueError): + sys.set_int_max_str_digits(-2) + with self.assertRaises(TypeError): + sys.set_int_max_str_digits(2_048.0) + @test.support.cpython_only @test.support.force_not_colorized_test_class diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index 199a9087dfe3bc1..a5e7bbbfad5ffca 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -666,21 +666,18 @@ async def f(): (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), (-2, 'line'), (-1, 'line'), (-1, 'return'), - (1, 'exception'), (2, 'line'), (1, 'line'), (-1, 'call'), diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index e6433f76a977b3b..e6f99581f0b7a66 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -576,6 +576,12 @@ def test_linux_ext_suffix(self): expected_suffixes = 'x86_64-linux-gnu.so', 'x86_64-linux-musl.so' self.assertEndsWith(suffix, expected_suffixes) + @unittest.skipIf(sysconfig.get_config_var('PY_BUILTIN_HASHLIB_HASHES') is None, + 'PY_BUILTIN_HASHLIB_HASHES required for this test') + def test_PY_BUILTIN_HASHLIB_HASHES_in_vars(self): + vars = sysconfig.get_config_vars() + self.assertFalse(vars['PY_BUILTIN_HASHLIB_HASHES'].startswith('"')) + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') def test_android_ext_suffix(self): machine = platform.machine() diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index e270cbb22e2d1a9..d657802d344803b 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -550,6 +550,53 @@ def test_extractfile_attrs(self): self.assertIs(fobj.seekable(), True) +class ReadSizeRecorder(io.BytesIO): + # Records the largest size ever passed to read(), so a test can check + # that tarfile does not request far more data than the archive holds + # (which on a real file would pre-allocate it). + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_read_size = 0 + + def read(self, size=-1): + if size is not None and size >= 0: + self.max_read_size = max(self.max_read_size, size) + return super().read(size) + + +@support.cpython_only +class ExtendedHeaderMemoryTest(unittest.TestCase): + # gh-151497: the size of a GNU long name/link or a pax extended header is + # read from the archive and is untrusted. A crafted header can claim a + # size far larger than the file actually contains; opening such an archive + # must not try to read (and so pre-allocate) the claimed size in one go. + + def crafted_archive(self, hdrtype): + tarinfo = tarfile.TarInfo("A") + tarinfo.type = hdrtype + tarinfo.size = 0xFFFFFFFF # ~4 GiB claimed in a 512-byte header + return tarinfo.tobuf(format=tarfile.GNU_FORMAT) + + def check(self, hdrtype): + fobj = ReadSizeRecorder(self.crafted_archive(hdrtype)) + try: + with tarfile.open(fileobj=fobj, mode="r:") as tar: + tar.getmembers() + except tarfile.ReadError: + pass # a truncated header is fine; we only check the allocation + # The bogus ~4 GiB size must never reach a single read() call. + self.assertLessEqual(fobj.max_read_size, tarfile._EXTHEADER_READ_CHUNK) + + def test_gnu_longname_oversized_size(self): + self.check(tarfile.GNUTYPE_LONGNAME) + + def test_gnu_longlink_oversized_size(self): + self.check(tarfile.GNUTYPE_LONGLINK) + + def test_pax_header_oversized_size(self): + self.check(tarfile.XHDTYPE) + + class MiscReadTestBase(CommonReadTest): is_stream = False @@ -893,10 +940,39 @@ def test_extractall_hardlink_on_symlink(self): self._assert_on_file_content(hardlink_filepath, sha256_regtype) +class GzipReadTestBase: + + def test_read_with_extra_field(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 4 + data[10:10] = b'\x05\x00extra' + with open(tmpname, 'wb') as f: + f.write(data) + print(self.mode) + with tarfile.open(tmpname, mode=self.mode): + pass + + def test_read_with_file_comment(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 16 + i = data.index(0, 10) + 1 + data[i:i] = b'comment\x00' + with open(tmpname, 'wb') as f: + f.write(data) + with tarfile.open(tmpname, mode=self.mode): + pass + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None -class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): +class GzipMiscReadTest(GzipTest, GzipReadTestBase, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): @@ -970,7 +1046,7 @@ def test_compare_members(self): finally: tar1.close() -class GzipStreamReadTest(GzipTest, StreamReadTest): +class GzipStreamReadTest(GzipTest, GzipReadTestBase, StreamReadTest): pass class Bz2StreamReadTest(Bz2Test, StreamReadTest): @@ -1313,6 +1389,27 @@ def _fs_supports_holes(): else: return False + def test_gnulong_dirname_strips_all_trailing_slashes(self): + # gh-149980: _proc_gnulong must normalize trailing slashes the same + # way _frombuf and _proc_builtin do (rstrip, not removesuffix), so + # a GNU long-name directory entry agrees with a short-name one. + long_name = "a" * 120 + "///" # > 100 bytes => GNUTYPE_LONGNAME + short_name = "b" * 20 + "///" + + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w", + format=tarfile.GNU_FORMAT) as tar: + for name in (short_name, long_name): + info = tarfile.TarInfo(name=name) + info.type = tarfile.DIRTYPE + tar.addfile(info) + + buf.seek(0) + with tarfile.open(fileobj=buf, mode="r") as tar: + names = [m.name for m in tar.getmembers()] + + self.assertEqual(names, ["b" * 20, "a" * 120]) + class PaxReadTest(LongnameTest, ReadTest, unittest.TestCase): @@ -3205,7 +3302,11 @@ def root_is_uid_gid_0(): import pwd, grp except ImportError: return False - if pwd.getpwuid(0)[0] != 'root': + try: + if pwd.getpwuid(0)[0] != 'root': + return False + except KeyError: + # On Cygwin, there is no root user (uid 0) return False if grp.getgrgid(0)[0] != 'root': return False @@ -3911,10 +4012,19 @@ def test_parent_symlink(self): + "which is outside the destination") with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'parent' would link to ['"].*outerdir['"], """ - + "which is outside the destination") + if self.dotdot_resolves_early: + # 'current/../..' normalises to '..', which is rejected. + self.expect_exception( + tarfile.LinkOutsideDestinationError, + """'parent' would link to ['"].*outerdir['"], """ + + "which is outside the destination") + else: + # 'current/..' normalises to '.'; the rewritten link is + # created and 'parent/evil' lands harmlessly inside the + # destination. + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='.') + self.expect_file('evil') else: # No symlink support. The symlinks are ignored. @@ -3976,6 +4086,9 @@ def test_realpath_limit_attack(self): check_flag=False)): if sys.platform == 'win32': self.expect_exception((FileNotFoundError, FileExistsError)) + elif sys.platform == 'cygwin': + exc = self.expect_exception(OSError) + self.assertEqual(exc.errno, errno.ELOOP) elif self.raised_exception: # Cannot symlink/hardlink: tarfile falls back to getmember() self.expect_exception(KeyError) @@ -3997,7 +4110,8 @@ def test_realpath_limit_attack(self): # 206: ERROR_FILENAME_EXCED_RANGE self.assertIn(exc.winerror, (3, 5, 206)) else: - self.assertEqual(exc.errno, errno.ENAMETOOLONG) + self.assertIn(exc.errno, + (errno.ENAMETOOLONG, errno.ELOOP)) @symlink_test def test_parent_symlink2(self): @@ -4174,6 +4288,76 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + @os_helper.skip_unless_symlink + def test_normpath_realpath_mismatch(self): + # The link-target check must validate the value that will actually + # be written to disk (the normalised linkname), not the original. + # Here 'a' is a symlink to a deep nonexistent path, so realpath() + # of 'a/../../...' stays inside the destination while normpath() + # collapses 'a/..' lexically and escapes. + depth = len(self.destdir.parts) + 5 + deep = '/'.join(f'p{i}' for i in range(depth)) + sneaky = 'a/' + '../' * depth + 'flag' + for kind in 'symlink_to', 'hardlink_to': + with self.subTest(kind): + with ArchiveMaker() as arc: + arc.add('a', symlink_to=deep) + arc.add('escape', **{kind: sneaky}) + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_symlink_trailing_slash(self): + # A trailing slash on a symlink member's name must not cause the + # link target to be resolved relative to the wrong directory. + with ArchiveMaker() as arc: + t = tarfile.TarInfo('x/') + t.type = tarfile.SYMTYPE + t.linkname = '..' + arc.tar_w.addfile(t) + arc.add('x/escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_link_at_destination(self): + # A link member whose name resolves to the destination directory + # itself must be rejected: otherwise the destination is replaced + # by a symlink and later members can be redirected through it. + for name in '', '.', './': + with ArchiveMaker() as arc: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = '.' + arc.tar_w.addfile(t) + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.OutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_empty_name_symlink_chain(self): + # Regression test for a chain of empty-named symlinks that + # incrementally redirects the destination outwards. + with ArchiveMaker() as arc: + for name, target in [('', ''), ('a/', '..'), + ('', 'dummy'), ('', 'a'), + ('b/', '..'), + ('', 'dummy'), ('', 'a/b')]: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = target + arc.tar_w.addfile(t) + arc.add('escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.FilterError) + @symlink_test def test_deep_symlink(self): # Test that symlinks and hardlinks inside a directory @@ -4276,6 +4460,30 @@ def test_sneaky_hardlink_fallback(self): self.expect_file("boom", symlink_to='../../link_here') self.expect_file("c", symlink_to='b') + @symlink_test + def test_sneaky_hardlink_fallback_deep(self): + # (CVE-2026-11940) + with ArchiveMaker() as arc: + arc.add("a/b/s", symlink_to=os.path.join("..", "escape")) + arc.add("s", hardlink_to=os.path.join("a", "b", "s")) + + with self.check_context(arc.open(), 'data'): + e = self.expect_exception( + tarfile.LinkFallbackError, + "link 's' would be extracted as a copy of " + + "'a/b/s', which was rejected") + self.assertIsInstance(e.__cause__, + tarfile.LinkOutsideDestinationError) + + for filter in 'tar', 'fully_trusted': + with self.subTest(filter), self.check_context(arc.open(), filter): + if not os_helper.can_symlink(): + self.expect_file("a/") + self.expect_file("a/b/") + else: + self.expect_file("a/b/s", symlink_to=os.path.join('..', 'escape')) + self.expect_file("s", symlink_to=os.path.join('..', 'escape')) + @symlink_test def test_exfiltration_via_symlink(self): # (CVE-2025-4138) @@ -4694,6 +4902,22 @@ def valueerror_filter(tarinfo, path): with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter): self.expect_exception(TypeError) # errorlevel is not int + @support.subTests('format', [tarfile.GNU_FORMAT, tarfile.PAX_FORMAT]) + def test_getmembers_big_size(self, format): + # gh-151981: A loop in seek() for streaming files tried to read the + # declared number of blocks even at EOF + tinfo = tarfile.TarInfo("huge-file") + tinfo.size = 1 << 64 + bio = io.BytesIO() + # Write header without data + bio.write(tinfo.tobuf(format)) + + # Reset & try to get contents + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r|") as tar: + with self.assertRaises(tarfile.ReadError): + tar.getmembers() + class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): testdir = os.path.join(TEMPDIR, "testoverwrite") diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 47450d3fd5976fa..70731d3222ced94 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -54,7 +54,11 @@ def test_eval_null_in_result(self): def test_eval_surrogates_in_result(self): tcl = self.interp - self.assertEqual(tcl.eval(r'set a "<\ud83d\udcbb>"'), '<\U0001f4bb>') + result = tcl.eval(r'set a "<\ud83d\udcbb>"') + if sys.platform == 'win32' and tcl_version >= (9, 0): + self.assertEqual('<\ud83d\udcbb>', result) + else: + self.assertEqual('<\U0001f4bb>', result) def testEvalException(self): tcl = self.interp @@ -289,7 +293,11 @@ def test_evalfile_surrogates_in_result(self): set b "<\\ud83d\\udcbb>" """) tcl.evalfile(filename) - self.assertEqual(tcl.eval('set b'), '<\U0001f4bb>') + result = tcl.eval('set b') + if sys.platform == 'win32' and tcl_version >= (9, 0): + self.assertEqual('<\ud83d\udcbb>', result) + else: + self.assertEqual('<\U0001f4bb>', result) def testEvalFileException(self): tcl = self.interp diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index b2b5390af33b005..3b081ecd4a3aa50 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -332,7 +332,9 @@ def test_read_only_directory(self): with _inside_empty_temp_dir(): probe = os.path.join(tempfile.tempdir, 'probe') if os.name == 'nt': - cmd = ['icacls', tempfile.tempdir, '/deny', 'Everyone:(W)'] + # Use security identifier *S-1-1-0 instead + # of localized "Everyone" to not depend on the locale. + cmd = ['icacls', tempfile.tempdir, '/deny', '*S-1-1-0:(W)'] stdout = None if support.verbose > 1 else subprocess.DEVNULL subprocess.run(cmd, check=True, stdout=stdout) else: @@ -355,7 +357,9 @@ def test_read_only_directory(self): self.make_temp() finally: if os.name == 'nt': - cmd = ['icacls', tempfile.tempdir, '/grant:r', 'Everyone:(M)'] + # Use security identifier *S-1-1-0 instead + # of localized "Everyone" to not depend on the locale. + cmd = ['icacls', tempfile.tempdir, '/grant:r', '*S-1-1-0:(M)'] subprocess.run(cmd, check=True, stdout=stdout) else: os.chmod(tempfile.tempdir, oldmode) @@ -511,6 +515,8 @@ def test_noinherit(self): self.assertFalse(retval > 0, "child process reports failure %d"%retval) @unittest.skipUnless(has_textmode, "text mode not available") + @unittest.skipIf(sys.platform == "cygwin", + "truncate text mode is not supported on Cygwin") def test_textmode(self): # _mkstemp_inner can create files in text mode diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index ce8392a6ccdbd66..3dcf5bd13791dc1 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -1,3 +1,4 @@ +import contextlib import errno import os import sys @@ -11,6 +12,19 @@ termios = import_module('termios') +# Skip the test on ENOTTY error +@contextlib.contextmanager +def skip_enotty_error(testcase, func, platforms): + try: + yield + except termios.error as exc: + if (exc.args[0] == errno.ENOTTY + and sys.platform.startswith(platforms)): + testcase.skipTest(f'termios.{func}() is not supported ' + f'with pseudo-terminals (?) on {sys.platform}') + raise + + @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") class TestFunctions(unittest.TestCase): @@ -90,7 +104,8 @@ def test_tcsetattr_errors(self): self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, attrs2) self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW, object()) self.assertRaises(TypeError, termios.tcsetattr, self.fd, termios.TCSANOW) - self.assertRaisesTermiosError(errno.EINVAL, termios.tcsetattr, self.fd, -1, attrs) + if sys.platform != 'cygwin': + self.assertRaisesTermiosError(errno.EINVAL, termios.tcsetattr, self.fd, -1, attrs) self.assertRaises(OverflowError, termios.tcsetattr, self.fd, 2**1000, attrs) self.assertRaises(TypeError, termios.tcsetattr, self.fd, object(), attrs) self.assertRaisesTermiosError(errno.ENOTTY, termios.tcsetattr, self.bad_fd, termios.TCSANOW, attrs) @@ -101,14 +116,10 @@ def test_tcsetattr_errors(self): @support.skip_android_selinux('tcsendbreak') def test_tcsendbreak(self): - try: + with skip_enotty_error(self, 'tcsendbreak', + ('freebsd', 'netbsd', 'cygwin')): termios.tcsendbreak(self.fd, 1) - except termios.error as exc: - if exc.args[0] == errno.ENOTTY and sys.platform.startswith(('freebsd', "netbsd")): - self.skipTest('termios.tcsendbreak() is not supported ' - 'with pseudo-terminals (?) on this platform') - raise - termios.tcsendbreak(self.stream, 1) + termios.tcsendbreak(self.stream, 1) @support.skip_android_selinux('tcsendbreak') def test_tcsendbreak_errors(self): @@ -123,8 +134,9 @@ def test_tcsendbreak_errors(self): @support.skip_android_selinux('tcdrain') def test_tcdrain(self): - termios.tcdrain(self.fd) - termios.tcdrain(self.stream) + with skip_enotty_error(self, 'tcdrain', ('cygwin',)): + termios.tcdrain(self.fd) + termios.tcdrain(self.stream) @support.skip_android_selinux('tcdrain') def test_tcdrain_errors(self): @@ -149,6 +161,7 @@ def test_tcflush_errors(self): self.assertRaises(TypeError, termios.tcflush, object(), termios.TCIFLUSH) self.assertRaises(TypeError, termios.tcflush, self.fd) + @unittest.skipIf(sys.platform == 'cygwin', 'test fails on Cygwin') def test_tcflush_clear_input_or_output(self): wfd = self.fd rfd = self.master_fd @@ -176,14 +189,16 @@ def test_tcflush_clear_input_or_output(self): @support.skip_android_selinux('tcflow') def test_tcflow(self): - termios.tcflow(self.fd, termios.TCOOFF) - termios.tcflow(self.fd, termios.TCOON) - termios.tcflow(self.fd, termios.TCIOFF) - termios.tcflow(self.fd, termios.TCION) + with skip_enotty_error(self, 'tcflow', ('cygwin',)): + termios.tcflow(self.fd, termios.TCOOFF) + termios.tcflow(self.fd, termios.TCOON) + termios.tcflow(self.fd, termios.TCIOFF) + termios.tcflow(self.fd, termios.TCION) @support.skip_android_selinux('tcflow') def test_tcflow_errors(self): - self.assertRaisesTermiosError(errno.EINVAL, termios.tcflow, self.fd, -1) + if sys.platform != 'cygwin': + self.assertRaisesTermiosError(errno.EINVAL, termios.tcflow, self.fd, -1) self.assertRaises(OverflowError, termios.tcflow, self.fd, 2**1000) self.assertRaises(TypeError, termios.tcflow, self.fd, object()) self.assertRaisesTermiosError(errno.ENOTTY, termios.tcflow, self.bad_fd, termios.TCOON) diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index 81f1a9c97393d12..a2a09f9de61490a 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -359,7 +359,7 @@ def test_main_with_time_unit(self): seconds_per_increment=0.003, switches=["-u", "parsec"] ) self.assertIn( - "choose from nsec, usec, msec, sec", error_stringio.getvalue() + "choose from 'nsec', 'usec', 'msec', 'sec'", error_stringio.getvalue() ) def test_main_exception(self): diff --git a/Lib/test/test_tkinter/test_dnd.py b/Lib/test/test_tkinter/test_dnd.py new file mode 100644 index 000000000000000..501b0d7f78586c5 --- /dev/null +++ b/Lib/test/test_tkinter/test_dnd.py @@ -0,0 +1,98 @@ +import unittest +import tkinter +from tkinter import dnd +from test.support import requires +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import AbstractTkTest + +requires('gui') + + +class Target: + def __init__(self, widget, log): + self.widget = widget + self.log = log + widget.dnd_accept = self.dnd_accept + + def dnd_accept(self, source, event): + self.log.append('accept') + return self + + def dnd_enter(self, source, event): + self.log.append('enter') + + def dnd_motion(self, source, event): + self.log.append('motion') + + def dnd_leave(self, source, event): + self.log.append('leave') + + def dnd_commit(self, source, event): + self.log.append('commit') + + +class Source: + def __init__(self, log): + self.log = log + + def dnd_end(self, target, event): + self.log.append('end') + + +class FakeEvent: + def __init__(self, widget, num=1): + self.num = num + self.widget = widget + self.x = self.y = self.x_root = self.y_root = 0 + + +class DndTest(AbstractTkTest, unittest.TestCase): + + def setUp(self): + super().setUp() + self.canvas = tkinter.Canvas(self.root) + self.canvas.pack() + # on_motion() locates the target with winfo_containing(). Bypass that + # real screen lookup, which depends on the window being visible and + # unobscured, so the test does not hinge on the window manager. + self.canvas.winfo_containing = lambda x, y: self.canvas + self.log = [] + self.source = Source(self.log) + self.target = Target(self.canvas, self.log) + + def test_drag_and_drop(self): + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + self.assertIsNotNone(handler) + handler.on_motion(FakeEvent(self.canvas)) # Enter the target. + handler.on_motion(FakeEvent(self.canvas)) # Move within the target. + handler.on_release(FakeEvent(self.canvas)) # Drop on the target. + self.assertEqual(self.log, + ['accept', 'enter', 'accept', 'motion', 'commit', 'end']) + + def test_cancel(self): + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + handler.on_motion(FakeEvent(self.canvas)) # Enter the target. + handler.cancel() # Leaves the target without committing. + self.assertEqual(self.log, ['accept', 'enter', 'leave', 'end']) + + def test_no_target(self): + # Nothing under the pointer accepts the drag. + self.canvas.winfo_containing = lambda x, y: None + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + handler.on_motion(FakeEvent(self.canvas)) + handler.on_release(FakeEvent(self.canvas)) + self.assertEqual(self.log, ['end']) + + def test_no_recursive_start(self): + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + self.assertIsNotNone(handler) + # A drag is already in progress, so a second start is ignored. + self.assertIsNone(dnd.dnd_start(self.source, FakeEvent(self.canvas))) + handler.cancel() + + def test_high_button_number_ignored(self): + self.assertIsNone(dnd.dnd_start(self.source, FakeEvent(self.canvas, num=6))) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tkinter/test_filedialog.py b/Lib/test/test_tkinter/test_filedialog.py new file mode 100644 index 000000000000000..054e719a0f883d9 --- /dev/null +++ b/Lib/test/test_tkinter/test_filedialog.py @@ -0,0 +1,73 @@ +import os +import unittest +from tkinter import filedialog +from tkinter.commondialog import Dialog +from test.support import requires, swap_attr +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import AbstractTkTest + +requires('gui') + + +class NativeDialogTest(AbstractTkTest, unittest.TestCase): + # Open, SaveAs and Directory wrap modal native dialogs. The _test_callback + # seam is called by show() just before the dialog would open; replacing it + # with a function that raises exercises show() without blocking on the + # dialog. + + def check(self, dialog_class, command): + self.assertEqual(dialog_class.command, command) + master = None + def test_callback(dialog, parent): + nonlocal master + master = parent + raise ZeroDivisionError + with swap_attr(Dialog, '_test_callback', test_callback): + d = dialog_class(self.root, title='Test') + self.assertRaises(ZeroDivisionError, d.show) + self.assertIs(master, self.root) + + def test_open(self): + self.check(filedialog.Open, 'tk_getOpenFile') + + def test_saveas(self): + self.check(filedialog.SaveAs, 'tk_getSaveFile') + + def test_directory(self): + self.check(filedialog.Directory, 'tk_chooseDirectory') + + +class FileDialogTest(AbstractTkTest, unittest.TestCase): + # The pure-Python FileDialog runs its own modal loop in go(); its logic is + # exercised here without entering the loop. + + def test_selection(self): + d = filedialog.FileDialog(self.root) + d.directory = os.path.abspath(os.sep) + d.set_selection('spam.txt') + self.assertEqual(os.path.basename(d.get_selection()), 'spam.txt') + + def test_filter(self): + d = filedialog.FileDialog(self.root) + d.set_filter(os.getcwd(), '*.py') + self.assertEqual(d.get_filter(), (os.getcwd(), '*.py')) + + def test_ok_cancel(self): + d = filedialog.FileDialog(self.root) + d.directory = os.getcwd() + d.set_selection('spam.txt') + d.ok_command() # Accepts the current selection. + self.assertEqual(os.path.basename(d.how), 'spam.txt') + d.cancel_command() # Returns no selection. + self.assertIsNone(d.how) + + def test_subclasses(self): + for cls in filedialog.LoadFileDialog, filedialog.SaveFileDialog: + with self.subTest(cls=cls.__name__): + d = cls(self.root) + self.assertIsInstance(d, filedialog.FileDialog) + self.assertEqual(d.top.title(), cls.title) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tkinter/test_font.py b/Lib/test/test_tkinter/test_font.py index ee147710fbfc857..8adc9e1151db4ed 100644 --- a/Lib/test/test_tkinter/test_font.py +++ b/Lib/test/test_tkinter/test_font.py @@ -21,6 +21,7 @@ def setUpClass(cls): cls.font = font.Font(root=cls.root, name=fontname, exists=False) def test_configure(self): + self.assertEqual(self.font.config, self.font.configure) options = self.font.configure() self.assertGreaterEqual(set(options), {'family', 'size', 'weight', 'slant', 'underline', 'overstrike'}) @@ -36,6 +37,26 @@ def test_configure(self): self.assertIsInstance(options[key], sizetype) self.assertIsInstance(self.font.cget(key), sizetype) self.assertIsInstance(self.font[key], sizetype) + self.assertRaisesRegex(tkinter.TclError, 'bad option "-spam"', + self.font.cget, 'spam') + self.assertRaisesRegex(tkinter.TclError, 'bad option "-spam"', + self.font.configure, spam='x') + self.assertRaises(TypeError, self.font.cget) + self.assertRaises(TypeError, self.font.cget, 'size', 'weight') + + def test_copy(self): + f = font.Font(root=self.root, family='Times', size=10, weight='bold') + copied = f.copy() + self.assertIsInstance(copied, font.Font) + self.assertIsNot(copied, f) + self.assertNotEqual(copied.name, f.name) + self.assertEqual(copied.actual(), f.actual()) + # The copy is independent of the original. + sizetype = int if self.wantobjects else str + copied.configure(size=20) + self.assertEqual(f.cget('size'), sizetype(10)) + self.assertEqual(copied.cget('size'), sizetype(20)) + self.assertRaises(TypeError, f.copy, 'x') def test_unicode_family(self): family = 'MS \u30b4\u30b7\u30c3\u30af' @@ -60,6 +81,9 @@ def test_actual(self): for key in 'size', 'underline', 'overstrike': self.assertIsInstance(options[key], sizetype) self.assertIsInstance(self.font.actual(key), sizetype) + self.assertRaisesRegex(tkinter.TclError, 'bad option "-spam"', + self.font.actual, 'spam') + self.assertRaises(TypeError, self.font.actual, 'size', 'weight', 'slant') def test_name(self): self.assertEqual(self.font.name, fontname) @@ -83,6 +107,11 @@ def test_equality(self): def test_measure(self): self.assertIsInstance(self.font.measure('abc'), int) + self.assertEqual(self.font.measure(''), 0) + self.assertIsInstance( + self.font.measure('abc', displayof=self.root), int) + self.assertRaises(TypeError, self.font.measure) + self.assertRaises(TypeError, self.font.measure, 'a', 'b', 'c') def test_metrics(self): metrics = self.font.metrics() @@ -90,8 +119,12 @@ def test_metrics(self): {'ascent', 'descent', 'linespace', 'fixed'}) for key in metrics: self.assertEqual(self.font.metrics(key), metrics[key]) + self.assertEqual(self.font.metrics(key, displayof=self.root), + metrics[key]) self.assertIsInstance(metrics[key], int) self.assertIsInstance(self.font.metrics(key), int) + self.assertRaisesRegex(tkinter.TclError, 'bad metric "-spam"', + self.font.metrics, 'spam') def test_families(self): families = font.families(self.root) diff --git a/Lib/test/test_tkinter/test_geometry_managers.py b/Lib/test/test_tkinter/test_geometry_managers.py index 3dcdadd1aacf10a..7fb0abff0049b5d 100644 --- a/Lib/test/test_tkinter/test_geometry_managers.py +++ b/Lib/test/test_tkinter/test_geometry_managers.py @@ -289,6 +289,25 @@ def test_pack_slaves(self): b.pack_configure() self.assertEqual(pack.pack_slaves(), [a, b]) + def test_pack_short_aliases(self): + # slaves, content and propagate are aliases of the pack_* methods + # (Misc precedes Pack, Place and Grid in the method resolution order). + pack, a, b, c, d = self.create2() + self.assertEqual(pack.slaves, pack.pack_slaves) + self.assertEqual(pack.content, pack.pack_content) + self.assertEqual(pack.propagate, pack.pack_propagate) + + self.assertEqual(pack.slaves(), []) + a.pack_configure() + self.assertEqual(pack.slaves(), [a]) + self.assertEqual(pack.content(), [a]) + + pack.configure(width=300, height=200) + pack.propagate(False) + self.root.update() + self.assertEqual(pack.winfo_reqwidth(), 300) + self.assertEqual(pack.winfo_reqheight(), 200) + class PlaceTest(AbstractWidgetTest, unittest.TestCase): @@ -503,6 +522,18 @@ def test_place_slaves(self): with self.assertRaises(TypeError): foo.place_slaves(0) + def test_place_method_aliases(self): + # The Place manager defines configure, info, forget, slaves and + # content as aliases of its place_* methods. On a real widget the + # short names are provided by Misc and Pack (earlier in the method + # resolution order), so the aliases are checked on the class itself. + self.assertIs(tkinter.Place.configure, tkinter.Place.place_configure) + self.assertIs(tkinter.Place.config, tkinter.Place.place_configure) + self.assertIs(tkinter.Place.info, tkinter.Place.place_info) + self.assertIs(tkinter.Place.forget, tkinter.Place.place_forget) + self.assertIs(tkinter.Place.slaves, tkinter.Misc.place_slaves) + self.assertIs(tkinter.Place.content, tkinter.Misc.place_content) + class GridTest(AbstractWidgetTest, unittest.TestCase): @@ -938,6 +969,30 @@ def test_grid_slaves(self): self.assertEqual(self.root.grid_slaves(column=1), [d, c, a]) self.assertEqual(self.root.grid_slaves(row=1, column=1), [d, c]) + def test_grid_short_aliases(self): + # columnconfigure, rowconfigure, size, anchor and bbox are aliases of + # the corresponding grid_* methods (Misc precedes Pack, Place and Grid + # in the method resolution order). + root = self.root + self.assertEqual(root.columnconfigure, root.grid_columnconfigure) + self.assertEqual(root.rowconfigure, root.grid_rowconfigure) + self.assertEqual(root.size, root.grid_size) + self.assertEqual(root.anchor, root.grid_anchor) + self.assertEqual(root.bbox, root.grid_bbox) + + self.assertEqual(root.size(), (0, 0)) + b = tkinter.Button(root) + b.grid_configure(column=2, row=3) + self.assertEqual(root.size(), (3, 4)) + + root.columnconfigure(0, weight=2) + self.assertEqual(root.grid_columnconfigure(0, 'weight'), 2) + root.rowconfigure(0, weight=3) + self.assertEqual(root.grid_rowconfigure(0, 'weight'), 3) + + root.anchor('se') + self.assertEqual(root.tk.call('grid', 'anchor', root), 'se') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index 3aca9515a33b248..4ce663406203339 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -288,6 +288,11 @@ def test_configure_width_height(self): image.configure(height=10) self.assertEqual(image['width'], '20') self.assertEqual(image['height'], '10') + self.assertEqual(image.cget('width'), image['width']) + self.assertEqual(image.cget('height'), image['height']) + self.assertRaises(TypeError, image.cget) + self.assertRaises(TypeError, image.cget, 'width', 'height') + self.assertEqual(image.config, image.configure) self.assertEqual(image.width(), 20) self.assertEqual(image.height(), 10) @@ -306,6 +311,12 @@ def test_blank(self): self.assertEqual(image.height(), 16) self.assertEqual(image.get(4, 6), self.colorlist(0, 0, 0)) + def test_redither(self): + image = self.create() + pixel = image.get(4, 6) + image.redither() # Recalculates the dithering; the data is unchanged. + self.assertEqual(image.get(4, 6), pixel) + def test_copy(self): image = self.create() image2 = image.copy() @@ -504,6 +515,16 @@ def test_put(self): self.assertEqual(image.get(0, 1), self.colorlist(0, 0, 255)) self.assertEqual(image.get(1, 1), self.colorlist(255, 255, 0)) + def test_put_format(self): + image = self.create() + with open(self.testfile, 'rb') as f: + data = f.read() + image2 = tkinter.PhotoImage(master=self.root) + image2.put(data, format='gif') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + def test_get(self): image = self.create() self.assertEqual(image.get(4, 6), self.colorlist(62, 116, 162)) @@ -514,6 +535,29 @@ def test_get(self): self.assertRaises(tkinter.TclError, image.get, 16, 15) self.assertRaises(tkinter.TclError, image.get, 15, 16) + @requires_tk(9, 0) + def test_get_withalpha(self): + image = self.create() + rgb = image.get(4, 6) + rgba = image.get(4, 6, withalpha=True) + if self.wantobjects: + self.assertEqual(rgba[:3], rgb) + self.assertEqual(len(rgba), 4) + self.assertIn(rgba[3], (0, 255)) # GIF alpha is fully on or off + else: + self.assertTrue(rgba.startswith(rgb + ' ')) + + @requires_tk(9, 0) + def test_metadata(self): + image = self.create() + # The -metadata configuration option holds the image's metadata. + image.configure(metadata=('Comment', 'spam')) + self.assertIn('Comment', str(image.cget('metadata'))) + # put() and data() accept a metadata dictionary, passed to the image + # format driver. put() does not change the image's own metadata. + image.put('{red green} {blue yellow}', metadata={'Comment': 'spam'}) + self.assertTrue(image.data(metadata={'Comment': 'spam'})) + def test_read(self): # Due to the Tk bug https://core.tcl-lang.org/tk/tktview/1576528 # the -from option does not work correctly for GIF and PNG files. @@ -656,6 +700,14 @@ def test_transparency(self): self.assertEqual(image.transparency_get(4, 6), True) image.transparency_set(4, 6, False) self.assertEqual(image.transparency_get(4, 6), False) + self.assertRaises(tkinter.TclError, image.transparency_get, -1, 0) + self.assertRaises(tkinter.TclError, image.transparency_get, 16, 0) + self.assertRaises(tkinter.TclError, image.transparency_set, -1, 0, True) + self.assertRaises(tkinter.TclError, image.transparency_set, 16, 0, True) + self.assertRaises(TypeError, image.transparency_get, 0) + self.assertRaises(TypeError, image.transparency_get, 0, 0, 0) + self.assertRaises(TypeError, image.transparency_set, 0, 0) + self.assertRaises(TypeError, image.transparency_set, 0, 0, True, 0) if __name__ == "__main__": diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index f017e94a8b3c222..98af4d822dadc43 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -1,13 +1,17 @@ import collections.abc import functools +import platform +import sys import unittest import tkinter from tkinter import TclError import enum from test import support +from test.support import os_helper from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import (AbstractTkTest, AbstractDefaultRootTest, - requires_tk, get_tk_patchlevel) + requires_tk, get_tk_patchlevel, + tcl_version) support.requires('gui') @@ -319,44 +323,233 @@ def test_clipboard_astral(self): with self.assertRaises(tkinter.TclError): root.clipboard_get() - def test_winfo_rgb(self): + def test_getint(self): + self.assertEqual(self.root.getint('42'), 42) + self.assertEqual(self.root.getint(42), 42) + self.assertEqual(self.root.getint('-5'), -5) + self.assertRaises(ValueError, self.root.getint, 'spam') - def assertApprox(col1, col2): - # A small amount of flexibility is required (bpo-45496) - # 33 is ~0.05% of 65535, which is a reasonable margin - for col1_channel, col2_channel in zip(col1, col2): - self.assertAlmostEqual(col1_channel, col2_channel, delta=33) + def test_getdouble(self): + self.assertEqual(self.root.getdouble('3.5'), 3.5) + self.assertEqual(self.root.getdouble(3), 3.0) + self.assertRaises(ValueError, self.root.getdouble, 'spam') - root = self.root - rgb = root.winfo_rgb + def test_getvar(self): + self.root.setvar('test_var', 'hello') + self.assertEqual(self.root.getvar('test_var'), 'hello') - # Color name. - self.assertEqual(rgb('red'), (65535, 0, 0)) - self.assertEqual(rgb('dark slate blue'), (18504, 15677, 35723)) - # #RGB - extends each 4-bit hex value to be 16-bit. - self.assertEqual(rgb('#F0F'), (0xFFFF, 0x0000, 0xFFFF)) - # #RRGGBB - extends each 8-bit hex value to be 16-bit. - assertApprox(rgb('#4a3c8c'), (0x4a4a, 0x3c3c, 0x8c8c)) - # #RRRRGGGGBBBB - assertApprox(rgb('#dede14143939'), (0xdede, 0x1414, 0x3939)) - # Invalid string. - with self.assertRaises(tkinter.TclError): - rgb('#123456789a') - # RGB triplet is invalid input. - with self.assertRaises(tkinter.TclError): - rgb((111, 78, 55)) + def test_register(self): + result = [] + def callback(): + result.append(1) + return 'spam' + name = self.root.register(callback) + self.assertIsInstance(name, str) + self.assertEqual(self.root.tk.call(name), 'spam') + self.assertEqual(result, [1]) + self.root.deletecommand(name) + self.assertRaises(TclError, self.root.tk.call, name) + + def test_option(self): + self.addCleanup(self.root.option_clear) + self.root.option_add('*Button.background', 'red') + b = tkinter.Button(self.root) + self.assertEqual(b.option_get('background', 'Background'), 'red') + self.assertEqual(b.option_get('foreground', 'Foreground'), '') + self.root.option_clear() + self.assertEqual(b.option_get('background', 'Background'), '') + + def test_option_readfile(self): + self.addCleanup(self.root.option_clear) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, 'w') as f: + f.write('*Button.background: red\n') + self.root.option_readfile(os_helper.TESTFN) + b = tkinter.Button(self.root) + self.assertEqual(b.option_get('background', 'Background'), 'red') + self.assertRaises(TclError, self.root.option_readfile, + os_helper.TESTFN + '.nonexistent') + self.assertRaises(TypeError, self.root.option_readfile) + self.assertRaises(TypeError, self.root.option_readfile, 'a', 'b', 'c') + + def test_nametowidget(self): + b = tkinter.Button(self.root, name='btn') + self.assertIs(self.root.nametowidget('btn'), b) + self.assertIs(self.root.nametowidget(str(b)), b) + self.assertRaises(KeyError, self.root.nametowidget, '.nonexistent') + + def test_focus_methods(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + f.focus_force() + self.root.update() + self.assertIs(self.root.focus_get(), f) + self.assertIs(self.root.focus_displayof(), f) + self.assertIs(f.focus_lastfor(), f) + b = tkinter.Button(f) + b.pack() + self.root.update() + b.focus_set() + self.root.update() + self.assertIs(self.root.focus_get(), b) - def test_winfo_pathname(self): - t = tkinter.Toplevel(self.root) - w = tkinter.Button(t) - wid = w.winfo_id() - self.assertIsInstance(wid, int) - self.assertEqual(self.root.winfo_pathname(hex(wid)), str(w)) - self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=None), str(w)) - self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=t), str(w)) - self.assertEqual(self.root.winfo_pathname(wid), str(w)) - self.assertEqual(self.root.winfo_pathname(wid, displayof=None), str(w)) - self.assertEqual(self.root.winfo_pathname(wid, displayof=t), str(w)) + def test_grab(self): + f = tkinter.Frame(self.root) + f.pack() + self.root.wait_visibility() + self.assertIsNone(self.root.grab_current()) + self.assertIsNone(f.grab_status()) + f.grab_set() + self.assertEqual(f.grab_status(), 'local') + self.assertIs(self.root.grab_current(), f) + f.grab_release() + self.assertIsNone(f.grab_status()) + self.assertIsNone(self.root.grab_current()) + + def test_selection_own(self): + self.root.selection_own() + self.assertIs(self.root.selection_own_get(), self.root) + f = tkinter.Frame(self.root) + f.selection_own() + self.assertIs(self.root.selection_own_get(), f) + + def test_event_add_delete_info(self): + self.addCleanup(self.root.event_delete, '<>') + self.root.event_add('<>', '', '') + self.assertEqual(self.root.event_info('<>'), + ('', '')) + self.assertIn('<>', self.root.event_info()) + self.root.event_delete('<>', '') + self.assertEqual(self.root.event_info('<>'), + ('',)) + self.root.event_delete('<>') + self.assertEqual(self.root.event_info('<>'), ()) + self.assertRaises(TypeError, self.root.event_delete) + + def test_bell(self): + self.root.bell() # No exception. + self.root.bell(displayof=self.root) + + def test_tk_focusNext_focusPrev(self): + f = tkinter.Frame(self.root) + f.pack() + entries = [tkinter.Entry(f) for _ in range(3)] + for entry in entries: + entry.pack() + # tk_focusNext skips widgets that are not viewable. + entries[-1].wait_visibility() + self.assertIs(entries[0].tk_focusNext(), entries[1]) + self.assertIs(entries[1].tk_focusNext(), entries[2]) + self.assertIs(entries[2].tk_focusPrev(), entries[1]) + self.assertIs(entries[1].tk_focusPrev(), entries[0]) + self.assertRaises(TypeError, entries[0].tk_focusNext, 'x') + self.assertRaises(TypeError, entries[0].tk_focusPrev, 'x') + + def test_tk_strictMotif(self): + self.addCleanup(self.root.tk_strictMotif, False) + self.assertIs(self.root.tk_strictMotif(), False) + self.assertIs(self.root.tk_strictMotif(True), True) + self.assertIs(self.root.tk_strictMotif(), True) + self.assertIs(self.root.tk_strictMotif(False), False) + self.assertRaises(TypeError, self.root.tk_strictMotif, 1, 2) + + def test_tk_bisque(self): + # tk_bisque resets the color palette; use a separate root so that + # the shared one is not affected. + root = tkinter.Tk() + self.addCleanup(root.destroy) + root.tk_bisque() + self.assertEqual(root['background'], '#ffe4c4') + self.assertRaises(TypeError, root.tk_bisque, 'x') + + def test_tk_appname(self): + old = self.root.tk_appname() + self.assertIsInstance(old, str) + self.addCleanup(self.root.tk_appname, old) + # Setting the name returns the actual name (possibly with a suffix + # appended to keep it unique). + new = self.root.tk_appname('PythonTkTest') + self.assertIsInstance(new, str) + self.assertEqual(self.root.tk_appname(), new) + + def test_tk_useinputmethods(self): + old = self.root.tk_useinputmethods() + self.assertIsInstance(old, bool) + self.addCleanup(self.root.tk_useinputmethods, old) + # Setting returns the resulting state. On systems without XIM support + # the state is always False, so only check the True->False direction. + self.assertIs(self.root.tk_useinputmethods(False), False) + + def test_tk_caret(self): + self.assertIsNone(self.root.tk_caret(x=5, y=10, height=20)) + caret = self.root.tk_caret() + if self.root._windowingsystem == 'aqua': + # macOS records the caret only for the key window. + self.assertEqual(set(caret), {'x', 'y', 'height'}) + else: + self.assertEqual(caret, {'x': 5, 'y': 10, 'height': 20}) + + def test_tk_scaling(self): + old = self.root.tk_scaling() + self.assertIsInstance(old, float) + self.assertGreater(old, 0) + self.addCleanup(self.root.tk_scaling, old) + # Setting the factor is reflected by a subsequent query. Tk may round + # it slightly when converting to and from its internal representation. + self.root.tk_scaling(2.0) + self.assertAlmostEqual(self.root.tk_scaling(), 2.0, delta=0.1) + + def test_tk_inactive(self): + ms = self.root.tk_inactive() + self.assertIsInstance(ms, int) + # A count of milliseconds, or -1 if the windowing system lacks support. + self.assertGreaterEqual(ms, -1) + # Resetting the timer returns None and does not raise. + self.assertIsNone(self.root.tk_inactive(reset=True)) + + def test_wait_variable(self): + var = tkinter.StringVar(self.root) + self.assertEqual(self.root.waitvar, self.root.wait_variable) + self.root.after(1, var.set, 'done') + self.root.wait_variable(var) # Returns once the variable is set. + self.assertEqual(var.get(), 'done') + + def test_wait_window(self): + top = tkinter.Toplevel(self.root) + self.root.after(1, top.destroy) + self.root.wait_window(top) # Returns once the window is destroyed. + self.assertFalse(top.winfo_exists()) + + def test_tk_focusFollowsMouse(self): + self.root.tk_focusFollowsMouse() # No exception. + + def test_selection_handle(self): + f = tkinter.Frame(self.root) + def handler(offset, length): + return 'PAYLOAD'[int(offset):int(offset) + int(length)] + f.selection_handle(handler) + f.selection_own() + self.assertEqual(f.selection_get(), 'PAYLOAD') + + def test_grab_set_global(self): + # A successful global grab directs all events on the display to this + # application, so only the error paths are tested here. + self.assertRaises(TypeError, self.root.grab_set_global, 'extra') + with self.subTest('non-viewable window'): + if self.root._windowingsystem != 'x11': + # Grabbing a non-viewable window fails only on X11; elsewhere + # it would actually grab the whole display. + self.skipTest('only X11 fails the grab') + f = tkinter.Frame(self.root) # not yet viewable + self.assertRaisesRegex(TclError, 'grab failed', f.grab_set_global) + + def test_send(self): + if self.root._windowingsystem != 'x11': + self.skipTest('send is only supported on X11') + self.assertRaisesRegex(TclError, 'no application named', + self.root.send, 'no_such_interp_xyzzy', 'set x 1') def test_event_repr_defaults(self): e = tkinter.Event() @@ -522,6 +715,180 @@ def test_iterable_protocol(self): widget in widget +class TkTest(AbstractTkTest, unittest.TestCase): + + def test_className(self): + # The className argument sets the class of the root window. Tk + # title-cases it: the first letter is upper-cased, the rest lower-cased. + cases = [ + ('fooBAR', 'Foobar'), + ('éÉ', 'Éé'), # small and capital E WITH ACUTE + ] + if tcl_version >= (9, 0): + # small and capital DESERET LETTER LONG I (a non-BMP script) + cases.append(('\U00010428\U00010400', '\U00010400\U00010428')) + for className, klass in cases: + root = tkinter.Tk(className=className) + try: + self.assertEqual(root.winfo_class(), klass) + finally: + root.destroy() + if tcl_version < (9, 0): + # gh-126219: title-casing a non-BMP first letter crashed Tcl 8.x; + # such a class name is now rejected. + self.assertRaises(ValueError, tkinter.Tk, className='\U00010428') + + +class WinfoTest(AbstractTkTest, unittest.TestCase): + + def test_winfo_rgb(self): + + def assertApprox(col1, col2): + # A small amount of flexibility is required (bpo-45496) + # 33 is ~0.05% of 65535, which is a reasonable margin + for col1_channel, col2_channel in zip(col1, col2): + self.assertAlmostEqual(col1_channel, col2_channel, delta=33) + + root = self.root + rgb = root.winfo_rgb + + # Color name. + self.assertEqual(rgb('red'), (65535, 0, 0)) + self.assertEqual(rgb('dark slate blue'), (18504, 15677, 35723)) + # #RGB - extends each 4-bit hex value to be 16-bit. + self.assertEqual(rgb('#F0F'), (0xFFFF, 0x0000, 0xFFFF)) + # #RRGGBB - extends each 8-bit hex value to be 16-bit. + assertApprox(rgb('#4a3c8c'), (0x4a4a, 0x3c3c, 0x8c8c)) + # #RRRRGGGGBBBB + assertApprox(rgb('#dede14143939'), (0xdede, 0x1414, 0x3939)) + # Invalid string. + with self.assertRaises(tkinter.TclError): + rgb('#123456789a') + # RGB triplet is invalid input. + with self.assertRaises(tkinter.TclError): + rgb((111, 78, 55)) + + def test_winfo_pathname(self): + t = tkinter.Toplevel(self.root) + w = tkinter.Button(t) + wid = w.winfo_id() + self.assertIsInstance(wid, int) + self.assertEqual(self.root.winfo_pathname(hex(wid)), str(w)) + self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=None), str(w)) + self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=t), str(w)) + self.assertEqual(self.root.winfo_pathname(wid), str(w)) + self.assertEqual(self.root.winfo_pathname(wid, displayof=None), str(w)) + self.assertEqual(self.root.winfo_pathname(wid, displayof=t), str(w)) + + def test_winfo_class_name_parent(self): + f = tkinter.Frame(self.root) + b = tkinter.Button(f) + self.assertEqual(f.winfo_class(), 'Frame') + self.assertEqual(b.winfo_class(), 'Button') + self.assertEqual(b.winfo_name(), str(b).rsplit('.', 1)[-1]) + self.assertEqual(b.winfo_parent(), str(f)) + self.assertEqual(f.winfo_parent(), str(self.root)) + self.assertIs(f.winfo_toplevel(), self.root) + t = tkinter.Toplevel(self.root) + self.assertIs(tkinter.Button(t).winfo_toplevel(), t) + self.assertEqual(self.root.nametowidget(b.winfo_parent()), f) + + def test_winfo_children(self): + self.assertEqual(self.root.winfo_children(), []) + f = tkinter.Frame(self.root) + b = tkinter.Button(f) + self.assertEqual(self.root.winfo_children(), [f]) + self.assertEqual(f.winfo_children(), [b]) + + def test_winfo_visual_info(self): + f = tkinter.Frame(self.root) + self.assertIsInstance(f.winfo_depth(), int) + self.assertIsInstance(f.winfo_cells(), int) + self.assertIsInstance(f.winfo_visual(), str) + self.assertIsInstance(f.winfo_visualid(), str) + self.assertIsInstance(f.winfo_colormapfull(), bool) + visuals = self.root.winfo_visualsavailable() + self.assertIsInstance(visuals, list) + for name, depth in visuals: + self.assertIsInstance(name, str) + self.assertIsInstance(depth, int) + + def test_winfo_viewable(self): + f = tkinter.Frame(self.root) + self.assertFalse(f.winfo_viewable()) + f.pack() + f.wait_visibility() + self.root.update() + self.assertTrue(f.winfo_viewable()) + + @requires_tk(9, 1) + def test_winfo_isdark(self): + self.assertIsInstance(self.root.winfo_isdark(), bool) + if self.root._windowingsystem == 'x11': + self.assertFalse(self.root.winfo_isdark()) + + def test_winfo_atom(self): + atom = self.root.winfo_atom('PRIMARY') + self.assertIsInstance(atom, int) + self.assertEqual(self.root.winfo_atomname(atom), 'PRIMARY') + self.assertEqual( + self.root.winfo_atomname(atom, displayof=self.root), 'PRIMARY') + self.assertEqual( + self.root.winfo_atom('PRIMARY', displayof=self.root), atom) + self.assertRaisesRegex(TclError, 'no atom exists', + self.root.winfo_atomname, 10 ** 9) + + def test_winfo_pointer(self): + self.assertIsInstance(self.root.winfo_pointerx(), int) + self.assertIsInstance(self.root.winfo_pointery(), int) + xy = self.root.winfo_pointerxy() + self.assertIsInstance(xy, tuple) + self.assertEqual(len(xy), 2) + self.assertTrue(all(isinstance(v, int) for v in xy)) + + def test_winfo_containing(self): + self.root.update() + # No window contains a point far off the screen. + self.assertIsNone(self.root.winfo_containing(-10000, -10000)) + self.assertIsNone( + self.root.winfo_containing(-10000, -10000, displayof=self.root)) + + def test_winfo_fpixels(self): + self.assertIsInstance(self.root.winfo_fpixels('1i'), float) + self.assertAlmostEqual(self.root.winfo_fpixels('1i'), + self.root.winfo_fpixels('72p')) + # Tk < 9 reports 'bad screen distance "spam"', Tk 9 reports + # 'expected screen distance ... but got "spam"'. + self.assertRaisesRegex(TclError, + r'(bad|expected) screen distance.*"spam"', + self.root.winfo_fpixels, 'spam') + + def test_winfo_screen(self): + for name in ('winfo_screenwidth', 'winfo_screenheight', + 'winfo_screenmmwidth', 'winfo_screenmmheight', + 'winfo_screencells', 'winfo_screendepth'): + value = getattr(self.root, name)() + self.assertIsInstance(value, int) + self.assertGreater(value, 0) + self.assertIsInstance(self.root.winfo_screenvisual(), str) + self.assertIsInstance(self.root.winfo_screen(), str) + self.assertIsInstance(self.root.winfo_server(), str) + + def test_winfo_vroot(self): + for name in ('winfo_vrootwidth', 'winfo_vrootheight', + 'winfo_vrootx', 'winfo_vrooty'): + self.assertIsInstance(getattr(self.root, name)(), int) + + def test_winfo_interps(self): + interps = self.root.winfo_interps() + self.assertIsInstance(interps, tuple) + # The registry of interpreters is only populated where "send" is + # supported (i.e. X11), so do not require this interpreter's name. + if self.root._windowingsystem == 'x11': + self.assertIn(self.root.tk.call('tk', 'appname'), interps) + self.assertEqual(self.root.winfo_interps(displayof=self.root), interps) + + class WmTest(AbstractTkTest, unittest.TestCase): def test_wm_attribute(self): @@ -578,12 +945,27 @@ def test_wm_attribute(self): def test_wm_iconbitmap(self): t = tkinter.Toplevel(self.root) + patchlevel = get_tk_patchlevel(t) + + if ( + t._windowingsystem == 'aqua' + and sys.platform == 'darwin' + and platform.machine() == 'x86_64' + and platform.mac_ver()[0].startswith('26.') + and ( + patchlevel[:3] <= (8, 6, 17) + or (9, 0) <= patchlevel[:3] <= (9, 0, 3) + ) + ): + # https://github.com/python/cpython/issues/146531 + # Tk bug 4a2070f0d3a99aa412bc582d386d575ca2f37323 + self.skipTest('wm iconbitmap hangs on macOS 26 Intel') + self.assertEqual(t.wm_iconbitmap(), '') t.wm_iconbitmap('hourglass') bug = False if t._windowingsystem == 'aqua': # Tk bug 13ac26b35dc55f7c37f70b39d59d7ef3e63017c8. - patchlevel = get_tk_patchlevel(t) if patchlevel < (8, 6, 17) or (9, 0) <= patchlevel < (9, 0, 2): bug = True if not bug: @@ -601,6 +983,187 @@ def test_wm_iconbitmap(self): t.destroy() + def test_wm_iconphoto(self): + t = tkinter.Toplevel(self.root) + img = tkinter.PhotoImage(master=t, width=16, height=16) + t.wm_iconphoto(False, img) # No exception. + t.wm_iconphoto(True, img) + self.assertRaises(tkinter.TclError, t.wm_iconphoto, False, 'spam') + + def test_wm_title(self): + t = tkinter.Toplevel(self.root) + t.title('Hello') + self.assertEqual(t.title(), 'Hello') + self.assertEqual(t.wm_title(), 'Hello') + t.wm_title('Spam') + self.assertEqual(t.title(), 'Spam') + + def test_wm_geometry(self): + t = tkinter.Toplevel(self.root) + t.geometry('200x100+10+20') + t.update() + self.assertRegex(t.geometry(), r'^200x100\+-?\d+\+-?\d+$') + self.assertEqual(t.wm_geometry(), t.geometry()) + + def test_wm_minsize_maxsize(self): + t = tkinter.Toplevel(self.root) + # Use a width above the minimum enforced by some platforms (72 on Aqua). + t.minsize(150, 100) + self.assertEqual(t.minsize(), (150, 100)) + t.maxsize(500, 600) + self.assertEqual(t.maxsize(), (500, 600)) + + def test_wm_resizable(self): + t = tkinter.Toplevel(self.root) + t.resizable(False, True) + self.assertEqual(t.resizable(), (0, 1)) + self.assertRaisesRegex(TclError, 'expected boolean value', + t.resizable, 'spam', True) + + def test_wm_aspect(self): + t = tkinter.Toplevel(self.root) + self.assertEqual(t.aspect(), None) + t.aspect(1, 2, 3, 4) + self.assertEqual(t.aspect(), (1, 2, 3, 4)) + + def test_wm_grid(self): + t = tkinter.Toplevel(self.root) + t.wm_grid(10, 10, 5, 5) + self.assertEqual(t.wm_grid(), (10, 10, 5, 5)) + + def test_wm_positionfrom_sizefrom(self): + # These set X11 size hints and may be no-ops on other platforms. + t = tkinter.Toplevel(self.root) + t.positionfrom('user') + self.assertIn(t.positionfrom(), ('user', '')) + t.sizefrom('program') + self.assertIn(t.sizefrom(), ('program', '')) + + def test_wm_focusmodel(self): + t = tkinter.Toplevel(self.root) + self.assertEqual(t.focusmodel(), 'passive') + t.focusmodel('active') + self.assertEqual(t.focusmodel(), 'active') + self.assertRaises(TclError, t.focusmodel, 'spam') + + def test_wm_iconname(self): + # WM_ICON_NAME is an X11 property and may be a no-op elsewhere. + t = tkinter.Toplevel(self.root) + t.iconname('Icon') + self.assertIn(t.iconname(), ('Icon', '')) + + def test_wm_iconposition(self): + t = tkinter.Toplevel(self.root) + t.wm_iconposition(3, 4) # An X11 hint; may be a no-op elsewhere. + if t._windowingsystem == 'x11': + self.assertEqual(t.wm_iconposition(), (3, 4)) + + def test_wm_iconmask_iconwindow(self): + if self.root._windowingsystem != 'x11': + self.skipTest('iconmask and iconwindow are X11-specific') + t = tkinter.Toplevel(self.root) + t.wm_iconmask('gray50') # No exception. + icon = tkinter.Toplevel(self.root) + t.wm_iconwindow(icon) + self.assertEqual(str(t.wm_iconwindow()), str(icon)) + + def test_wm_colormapwindows(self): + if self.root._windowingsystem != 'x11': + self.skipTest('colormapwindows is X11-specific') + t = tkinter.Toplevel(self.root) + self.assertEqual(t.wm_colormapwindows(), []) + f = tkinter.Frame(t) + t.wm_colormapwindows(f) + self.assertEqual([str(w) for w in t.wm_colormapwindows()], [str(f)]) + + def test_wm_manage_forget(self): + f = tkinter.Frame(self.root) + self.root.wm_manage(f) # Make the frame a top-level window. + self.root.wm_forget(f) # Revert it; no exception either way. + + def test_wm_client_command(self): + t = tkinter.Toplevel(self.root) + t.client('myhost') + t.wm_command('myapp -x') + # WM_CLIENT_MACHINE and WM_COMMAND are X11 properties; elsewhere the + # setters may be no-ops and wm_command may return a split list. + if t._windowingsystem == 'x11': + self.assertEqual(t.client(), 'myhost') + self.assertEqual(t.wm_command(), 'myapp -x') + + def test_wm_overrideredirect(self): + t = tkinter.Toplevel(self.root) + self.assertFalse(t.overrideredirect()) + t.overrideredirect(True) + self.assertTrue(t.overrideredirect()) + + def test_wm_state(self): + t = tkinter.Toplevel(self.root) + t.update() + self.assertEqual(t.state(), 'normal') + t.withdraw() + self.assertEqual(t.state(), 'withdrawn') + t.deiconify() + t.update() + self.assertEqual(t.state(), 'normal') + self.assertRaises(TclError, t.state, 'spam') + + def test_wm_frame(self): + t = tkinter.Toplevel(self.root) + t.update() + self.assertIsInstance(t.frame(), str) + + def test_wm_group(self): + # The window group is an X11 concept and may be a no-op elsewhere. + t = tkinter.Toplevel(self.root) + t.group(self.root) + self.assertIn(t.group(), (str(self.root), '')) + + def test_wm_protocol(self): + t = tkinter.Toplevel(self.root) + self.assertIsInstance(t.protocol(), tuple) + t.protocol('WM_SAVE_YOURSELF', lambda: None) + self.assertIn('WM_SAVE_YOURSELF', t.protocol()) + # Querying a single protocol returns the bound command name. + self.assertTrue(t.protocol('WM_SAVE_YOURSELF')) + + def test_wm_transient(self): + t = tkinter.Toplevel(self.root) + self.assertEqual(t.transient(), '') + t.transient(self.root) + self.assertEqual(str(t.transient()), str(self.root)) + + def test_wm_stackorder(self): + t1 = tkinter.Toplevel(self.root) + t2 = tkinter.Toplevel(self.root) + t1.deiconify() + t2.deiconify() + self.root.update() + t1.lift(t2) # Raise t1 above t2. + self.root.update() + order = self.root.wm_stackorder() + self.assertIsInstance(order, list) + self.assertTrue(all(isinstance(w, tkinter.Misc) for w in order)) + names = [str(w) for w in order] + self.assertIn(str(t1), names) + self.assertIn(str(t2), names) + # The list is ordered from lowest to highest, consistently with the + # isabove/isbelow queries. + self.assertGreater(names.index(str(t1)), names.index(str(t2))) + self.assertIs(t1.wm_stackorder('isabove', t2), True) + self.assertIs(t1.wm_stackorder('isbelow', t2), False) + self.assertIs(t2.wm_stackorder('isbelow', t1), True) + + @requires_tk(9, 0) + def test_wm_iconbadge(self): + if self.root._windowingsystem == 'x11': + # On X11 the badge requires ::tk::icons::base_icon to be set. + self.skipTest('iconbadge needs a base icon on X11') + # The badge is not queryable, so just check the call does not fail. + self.root.wm_iconbadge('3') + self.root.wm_iconbadge('!') + self.root.wm_iconbadge('') + class EventTest(AbstractTkTest, unittest.TestCase): @@ -1030,6 +1593,8 @@ def test3(e): pass self.assertNotIn(funcid, script) self.assertNotIn(funcid2, script) self.assertIn(funcid3, script) + self.assertCommandNotExist(funcid) + self.assertCommandNotExist(funcid2) self.assertCommandExist(funcid3) def test_bind_class(self): @@ -1074,8 +1639,8 @@ def test2(e): pass unbind_class('Test', event) self.assertEqual(bind_class('Test', event), '') self.assertEqual(bind_class('Test'), ()) - self.assertCommandExist(funcid) - self.assertCommandExist(funcid2) + self.assertCommandNotExist(funcid) + self.assertCommandNotExist(funcid2) unbind_class('Test', event) # idempotent @@ -1103,8 +1668,8 @@ def test3(e): pass self.assertNotIn(funcid, script) self.assertNotIn(funcid2, script) self.assertIn(funcid3, script) - self.assertCommandExist(funcid) - self.assertCommandExist(funcid2) + self.assertCommandNotExist(funcid) + self.assertCommandNotExist(funcid2) self.assertCommandExist(funcid3) def test_bind_all(self): @@ -1146,8 +1711,8 @@ def test2(e): pass unbind_all(event) self.assertEqual(bind_all(event), '') self.assertNotIn(event, bind_all()) - self.assertCommandExist(funcid) - self.assertCommandExist(funcid2) + self.assertCommandNotExist(funcid) + self.assertCommandNotExist(funcid2) unbind_all(event) # idempotent @@ -1175,8 +1740,8 @@ def test3(e): pass self.assertNotIn(funcid, script) self.assertNotIn(funcid2, script) self.assertIn(funcid3, script) - self.assertCommandExist(funcid) - self.assertCommandExist(funcid2) + self.assertCommandNotExist(funcid) + self.assertCommandNotExist(funcid2) self.assertCommandExist(funcid3) def _test_tag_bind(self, w): diff --git a/Lib/test/test_tkinter/test_scrolledtext.py b/Lib/test/test_tkinter/test_scrolledtext.py new file mode 100644 index 000000000000000..913e7d8aab6c46a --- /dev/null +++ b/Lib/test/test_tkinter/test_scrolledtext.py @@ -0,0 +1,65 @@ +import unittest +import tkinter +from tkinter.scrolledtext import ScrolledText +from test.support import requires +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import AbstractTkTest + +requires('gui') + + +class ScrolledTextTest(AbstractTkTest, unittest.TestCase): + + def create(self, **kwargs): + st = ScrolledText(self.root, **kwargs) + self.addCleanup(st.destroy) + return st + + def test_create(self): + st = self.create(background='red', height=5) + # It is a Text widget held in a Frame together with a Scrollbar. + self.assertIsInstance(st, tkinter.Text) + self.assertIsInstance(st.frame, tkinter.Frame) + self.assertIsInstance(st.vbar, tkinter.Scrollbar) + self.assertEqual(st.winfo_parent(), str(st.frame)) + # str() returns the frame, so that geometry managers manage it. + self.assertEqual(str(st), str(st.frame)) + # Keyword options configure the Text. + self.assertEqual(str(st['background']), 'red') + self.assertEqual(st['height'], 5 if self.wantobjects else '5') + + def test_text_methods(self): + st = self.create() + st.insert('1.0', 'hello\nworld') + self.assertEqual(st.get('1.0', 'end-1c'), 'hello\nworld') + self.assertEqual(st.index('end-1c'), '2.5') + st.delete('1.0', 'end') + self.assertEqual(st.get('1.0', 'end-1c'), '') + + def test_geometry_methods(self): + st = self.create() + # configure is not redirected; it configures the Text. + st.configure(height=8) + self.assertEqual(st['height'], 8 if self.wantobjects else '8') + # Pack, Grid and Place methods are redirected to the frame. + st.pack() + self.root.update() + self.assertEqual(st.frame.winfo_manager(), 'pack') + self.assertEqual(st.pack_info(), st.frame.pack_info()) + st.pack_forget() + self.assertEqual(st.frame.winfo_manager(), '') + + def test_scrollbar(self): + st = self.create(height=5) + st.pack() + st.insert('1.0', '\n'.join(map(str, range(100)))) + self.root.update() + # The scrollbar tracks the text view. + self.assertEqual(st.vbar.get(), st.yview()) + st.yview_moveto(1.0) + self.root.update() + self.assertEqual(st.vbar.get()[1], 1.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tkinter/test_simpledialog.py b/Lib/test/test_tkinter/test_simpledialog.py index 313ad82e0a2c0d9..5c739d9ad6e6422 100644 --- a/Lib/test/test_tkinter/test_simpledialog.py +++ b/Lib/test/test_tkinter/test_simpledialog.py @@ -1,13 +1,403 @@ import unittest import tkinter +from tkinter import messagebox, ttk from test.support import requires, swap_attr from test.test_tkinter.support import setUpModule # noqa: F401 -from test.test_tkinter.support import AbstractDefaultRootTest -from tkinter.simpledialog import Dialog, askinteger +from test.test_tkinter.support import AbstractDefaultRootTest, AbstractTkTest +from tkinter.simpledialog import (Dialog, SimpleDialog, + askinteger, askfloat, askstring, + _QueryInteger, _QueryFloat, _QueryString, + _underline_ampersand, _find_alt_key_target) requires('gui') +class SimpleDialogTest(AbstractTkTest, unittest.TestCase): + # SimpleDialog's modal loop is in go(); its bindings are exercised here by + # generating events on the constructed dialog, without entering the loop. + + def create(self, **kw): + kw.setdefault('text', 'Question?') + kw.setdefault('buttons', ['Yes', 'No']) + kw.setdefault('default', 0) + kw.setdefault('cancel', 1) + d = SimpleDialog(self.root, **kw) + self.addCleanup(lambda: d.root.winfo_exists() and d.root.destroy()) + return d + + # --- Widget set and appearance --- + + def test_use_ttk(self): + # By default SimpleDialog uses the themed (ttk) widgets (tk::MessageBox). + d = self.create(buttons=['OK'], bitmap='warning') + self.assertEqual(d._buttons[0].winfo_class(), 'TButton') + self.assertEqual(d.message.winfo_class(), 'TLabel') + self.assertEqual(str(d.message.cget('anchor')), 'nw') # cf. MessageBox + # The standard icons are drawn with themed images (cf. MessageBox). + self.assertEqual(d.bitmap.winfo_class(), 'TLabel') + self.assertIn('::tk::icons::warning', str(d.bitmap.cget('image'))) + # tk::MessageBox makes the buttons equal width. + self.assertEqual( + str(d.root.children['bot'].grid_columnconfigure(0)['uniform']), + 'buttons') + # The dialog uses the themed background colour (cf. MessageBox). + self.assertEqual(str(d.root.cget('background')), + ttk.Style(d.root).lookup('.', 'background')) + # The bindings work with the themed buttons too. + d._buttons[0].focus_force() + d.root.update() + d.root.event_generate('') + d.root.update() + self.assertEqual(d.num, 0) + + def test_use_classic(self): + # use_ttk=False uses the classic Tk widgets, modelled on tk_dialog. + d = self.create(buttons=['OK'], bitmap='warning', use_ttk=False) + self.assertEqual(d._buttons[0].winfo_class(), 'Button') + self.assertEqual(d.message.winfo_class(), 'Label') + if d.root._windowingsystem == 'x11': + self.assertEqual(str(d.frame.cget('relief')), 'raised') + # tk_dialog does not make the buttons equal width. + self.assertIsNone(d.root.children['bot'].grid_columnconfigure(0)['uniform']) + # The bitmap is a classic monochrome label. + self.assertEqual(str(d.bitmap.cget('bitmap')), 'warning') + + def test_class_name(self): + # class_ sets the Tk class of the dialog window (default 'Dialog'). + self.assertEqual(self.create().root.winfo_class(), 'Dialog') + d = self.create(class_='MyDialog') + self.assertEqual(d.root.winfo_class(), 'MyDialog') + + # --- Message, detail and bitmap content --- + + def test_no_detail(self): + # Without a detail message the message label expands. + d = self.create() + self.assertIsNone(d.detail) + self.assertEqual(int(d.message.pack_info()['expand']), 1) + + def test_detail(self): + # The detail message is shown below the main message. + d = self.create(detail='More information.') + self.assertEqual(d.detail.winfo_class(), 'TLabel') + self.assertEqual(str(d.detail.cget('text')), 'More information.') + self.assertEqual(str(d.detail.cget('anchor')), 'nw') # cf. MessageBox + # With a detail message it expands and the main message does not. + self.assertEqual(int(d.message.pack_info()['expand']), 0) + self.assertEqual(int(d.detail.pack_info()['expand']), 1) + + def test_bitmap_fallback(self): + # A non-standard bitmap has no themed image, so even the ttk version + # falls back to a classic bitmap label. + d = self.create(buttons=['OK'], bitmap='questhead') + self.assertEqual(d.bitmap.winfo_class(), 'Label') + self.assertEqual(str(d.bitmap.cget('bitmap')), 'questhead') + + def test_bitmap_detail_layout(self): + # The bitmap is packed first to claim the whole left side; the message + # and detail stack on its right, as in tk::MessageBox. + d = self.create(buttons=['OK'], bitmap='warning', detail='More.') + top = d.root.children['top'] + layout = [(w.winfo_name(), str(w.pack_info()['side'])) + for w in top.pack_slaves()] + self.assertEqual(layout, + [('bitmap', 'left'), ('msg', 'top'), ('dtl', 'top')]) + + # --- Buttons and keyboard --- + + def test_button_options(self): + # A button entry can be a mapping of options, not just a label (like + # the "[name ?-option value ...?]" button specs in tk::MessageBox). + d = self.create(buttons=['Yes', + {'text': 'No', 'underline': 0, 'width': 12}]) + yes, no = d._buttons + self.assertEqual(str(yes.cget('text')), 'Yes') + self.assertEqual(str(no.cget('text')), 'No') + self.assertEqual(int(no.cget('underline')), 0) + self.assertEqual(str(no.cget('width')), '12') + # The dialog still controls the default ring (default=0) ... + self.assertEqual(str(yes.cget('default')), 'active') + self.assertEqual(str(no.cget('default')), 'normal') + # ... and the command, which records the button index. + no.invoke() + self.assertEqual(d.num, 1) + + def test_default_ring(self): + # The default ring follows the keyboard focus among the buttons + # (cf. tk::MessageBox). + d = self.create() # buttons ['Yes', 'No'], default 0 + b0, b1 = d._buttons + self.assertEqual(str(b1.cget('default')), 'normal') + b1.focus_force() + d.root.update() + self.assertEqual(str(b1.cget('default')), 'active') # focused -> ring + b0.focus_force() + d.root.update() + self.assertEqual(str(b1.cget('default')), 'normal') # unfocused -> none + + def test_alt_key(self): + # Alt + an underlined character (the "underline" button option) invokes + # the matching button (cf. tk::AmpWidget in tk::MessageBox). + d = self.create(buttons=['Yes', {'text': 'No', 'underline': 0}]) + d._buttons[0].focus_force() + d.root.update() + d.root.event_generate('') # "No" -> underline 0 -> "N" + d.root.update() + self.assertEqual(d.num, 1) + + def test_return_invokes_focused_button(self): + # invokes the button with the focus, even if it is not the + # default and the focus was not moved by keyboard traversal. + d = self.create(buttons=['Yes', 'No']) # default 0 + d._buttons[1].focus_force() + d.root.update() + d.root.event_generate('') + d.root.update() + self.assertEqual(d.num, 1) + + def test_focus_next_then_return(self): + # moves the focus to the next button; invokes it. + d = self.create(buttons=['Yes', 'No']) + d._buttons[0].focus_force() + d.root.update() + d._buttons[0].event_generate('') + d.root.update() + d.root.event_generate('') + d.root.update() + self.assertEqual(d.num, 1) + + def test_focus_prev_then_return(self): + # moves the focus to the previous button. + d = self.create(buttons=['Yes', 'No']) + d._buttons[1].focus_force() + d.root.update() + d._buttons[1].event_generate('') + d.root.update() + d.root.event_generate('') + d.root.update() + self.assertEqual(d.num, 0) + + def test_return_activates_default(self): + # with the focus off the buttons invokes the default button. + d = self.create() # default 0 + d.root.focus_force() # the dialog, not a button, has the focus + d.root.update() + d.root.event_generate('') + d.root.update() + self.assertEqual(d.num, 0) + + def test_return_no_default(self): + # With no default button, off the buttons rings the bell and + # leaves the dialog open instead of activating a button. + d = self.create(default=None) + d.root.focus_force() # the dialog, not a button, has the focus + d.root.update() + bells = [] + with swap_attr(d.root, 'bell', lambda *a, **k: bells.append(True)): + d.root.event_generate('') + d.root.update() + self.assertTrue(bells) # rang the bell + self.assertIsNone(d.num) + self.assertTrue(d.root.winfo_exists()) + + # --- Modal lifecycle --- + + def test_destroy_cancels(self): + # Destroying the window records the cancel index. + d = self.create() + d.root.update() + d.root.destroy() + self.assertEqual(d.num, 1) + + def test_wm_delete_cancels(self): + # Closing the window through the window manager records the cancel index. + d = self.create() # cancel 1 + d.wm_delete_window() + self.assertEqual(d.num, 1) + + def test_wm_delete_no_cancel(self): + # With no cancel index, closing the window through the window manager + # rings the bell and leaves the dialog open instead of recording an + # index. + d = self.create(default=None, cancel=None) + d.root.update() + bells = [] + with swap_attr(d.root, 'bell', lambda *a, **k: bells.append(True)): + d.wm_delete_window() + d.root.update() + self.assertTrue(bells) # rang the bell + self.assertIsNone(d.num) + self.assertTrue(d.root.winfo_exists()) + + def test_go(self): + # go() runs the modal loop and returns the chosen button's index. + d = self.create() + d.root.after(1, lambda: d._buttons[0].invoke()) + self.assertEqual(d.go(), 0) + + +class DialogTest(AbstractTkTest, unittest.TestCase): + # Dialog's button box is modelled on tk::MessageBox. + + def open(self, **kw): + with swap_attr(Dialog, 'wait_window', staticmethod(lambda w: None)): + d = _QueryInteger('Title', 'Prompt', parent=self.root, **kw) + self.addCleanup(lambda: d.winfo_exists() and d.destroy()) + return d + + # --- Widget set and appearance --- + + def test_use_ttk(self): + # The query dialogs use the themed (ttk) widgets by default. + d = self.open() + self.assertEqual(d.children['ok'].winfo_class(), 'TButton') + self.assertEqual(d.entry.winfo_class(), 'TEntry') + # tk::MessageBox makes the buttons equal width. + self.assertEqual( + str(d.children['bot'].grid_columnconfigure(0)['uniform']), 'buttons') + + def test_use_classic(self): + # use_ttk=False uses the classic Tk widgets, modelled on tk_dialog. + d = self.open(use_ttk=False) + self.assertEqual(d.children['ok'].winfo_class(), 'Button') + self.assertEqual(d.entry.winfo_class(), 'Entry') + if d._windowingsystem == 'x11': + self.assertEqual(str(d.children['bot'].cget('relief')), 'raised') + # tk_dialog does not make the buttons equal width. + self.assertIsNone(d.children['bot'].grid_columnconfigure(0)['uniform']) + # The bindings work with the classic buttons too. + invoked = [] + cancel = d.children['cancel'] + cancel.configure(command=lambda: invoked.append(True)) + cancel.focus_force() + d.update() + d.event_generate('') + d.update() + self.assertTrue(invoked) + + def test_background(self): + # The ttk dialog adopts the ttk background, even a customized one, + # while the classic dialog keeps the default Toplevel background. + style = ttk.Style(self.root) + old = style.lookup('.', 'background') + style.configure('.', background='#123456') + self.addCleanup(style.configure, '.', background=old) + d = self.open() + self.assertEqual(str(d.cget('background')), '#123456') + d = self.open(use_ttk=False) + ref = tkinter.Toplevel(self.root) + self.addCleanup(ref.destroy) + self.assertEqual(str(d.cget('background')), str(ref.cget('background'))) + + def test_base_classic_by_default(self): + # The Dialog base defaults to classic widgets so that subclasses adding + # classic widgets keep their look; only the query dialogs opt into ttk. + class MyDialog(Dialog): + def body(self, master): + pass + with swap_attr(Dialog, 'wait_window', staticmethod(lambda w: None)): + d = MyDialog(self.root, 'Title') + self.addCleanup(lambda: d.winfo_exists() and d.destroy()) + self.assertEqual(d.children['ok'].winfo_class(), 'Button') + + # --- Buttons and keyboard --- + + def test_button_default(self): + d = self.open() + self.assertEqual(str(d.children['ok'].cget('default')), 'active') + self.assertEqual(str(d.children['cancel'].cget('default')), 'normal') + + def test_underline_ampersand(self): + self.assertEqual(_underline_ampersand('Yes'), ('Yes', -1)) + self.assertEqual(_underline_ampersand('&Yes'), ('Yes', 0)) + self.assertEqual(_underline_ampersand('Save &As'), ('Save As', 5)) + self.assertEqual(_underline_ampersand('A&&B'), ('A&B', -1)) + self.assertEqual(_underline_ampersand('&a&b'), ('ab', 0)) + + def test_button_accelerator(self): + # The buttons' "&" accelerators are parsed (cf. tk::AmpWidget). + d = self.open() + ok = d.children['ok'] # "&OK" -> underline 0 -> "O" + self.assertEqual(str(ok.cget('text')), 'OK') + self.assertEqual(int(ok.cget('underline')), 0) + + def test_default_ring(self): + # The default ring follows the keyboard focus among the buttons. + d = self.open() + cancel = d.children['cancel'] + self.assertEqual(str(cancel.cget('default')), 'normal') + cancel.focus_force() + d.update() + self.assertEqual(str(cancel.cget('default')), 'active') + d.children['ok'].focus_force() + d.update() + self.assertEqual(str(cancel.cget('default')), 'normal') + + def test_find_alt_key_target(self): + d = self.open() + ok = d.children['ok'] # "&OK" -> "O" + cancel = d.children['cancel'] # "&Cancel" -> "C" + self.assertIs(_find_alt_key_target(d, 'o'), ok) + self.assertIs(_find_alt_key_target(d, 'O'), ok) # case-insensitive + self.assertIs(_find_alt_key_target(d, 'c'), cancel) + self.assertIsNone(_find_alt_key_target(d, 'q')) + + def test_alt_key(self): + # The accelerator key (Alt + the underlined letter) invokes the button: + # -> _alt_key -> the button's <> -> invoke. + d = self.open() + invoked = [] + cancel = d.children['cancel'] # "&Cancel" + cancel.configure(command=lambda: invoked.append(True)) + d.focus_force() + d.update() + d.event_generate('') + d.update() + self.assertTrue(invoked) + + def test_return_invokes_focused_button(self): + # invokes the focused button. + d = self.open() + invoked = [] + cancel = d.children['cancel'] + cancel.configure(command=lambda: invoked.append(True)) + cancel.focus_force() + d.update() + d.event_generate('') + d.update() + self.assertEqual(invoked, [True]) + + def test_focus_next_then_return(self): + # moves the focus to the next button; invokes it. + d = self.open() + invoked = [] + for name in ('ok', 'cancel'): + d.children[name].configure(command=lambda name=name: invoked.append(name)) + ok = d.children['ok'] + ok.focus_force() + d.update() + ok.event_generate('') # OK -> Cancel + d.update() + d.event_generate('') + d.update() + self.assertEqual(invoked, ['cancel']) + + def test_focus_prev_then_return(self): + # moves the focus to the previous button. + d = self.open() + invoked = [] + for name in ('ok', 'cancel'): + d.children[name].configure(command=lambda name=name: invoked.append(name)) + cancel = d.children['cancel'] + cancel.focus_force() + d.update() + cancel.event_generate('') # Cancel -> OK + d.update() + d.event_generate('') + d.update() + self.assertEqual(invoked, ['ok']) + + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): def test_askinteger(self): @@ -32,5 +422,131 @@ def mock_wait_window(w): self.assertRaises(RuntimeError, askinteger, "Go To Line", "Line number") +class QueryDialogTest(AbstractTkTest, unittest.TestCase): + # The query dialogs are modal: their __init__ blocks in wait_window(). + # Mock that out so the dialog stays alive and can be driven with generated + # events, exercising the / bindings and the validation. + + def open(self, query, **kw): + with swap_attr(Dialog, 'wait_window', staticmethod(lambda w: None)): + d = query("Title", "Prompt", parent=self.root, **kw) + self.addCleanup(lambda: d.winfo_exists() and d.destroy()) + d.focus_force() + d.update() + return d + + def enter(self, d, value, key=''): + d.entry.delete(0, 'end') + d.entry.insert(0, value) + d.event_generate(key) + d.update() + + # --- Prompt and entry --- + + def test_prompt_wraplength(self): + # A long prompt wraps instead of widening the dialog (cf. MessageBox). + d = self.open(_QueryInteger) + body = d.children['top'] + label = [w for w in body.winfo_children() if w is not d.entry][0] + self.assertEqual(str(label.cget('wraplength')), '3i') + + def test_prompt_column_expands(self): + # The prompt and entry column expands to the full width, like the + # weighted message column in tk::MessageBox. + d = self.open(_QueryInteger) + body = d.children['top'] + self.assertEqual(body.grid_columnconfigure(0)['weight'], 1) + + def test_initialvalue(self): + # The entry is pre-filled with the initial value, which is accepted. + d = self.open(_QueryInteger, initialvalue=42) + self.assertEqual(d.entry.get(), '42') + d.event_generate('') + d.update() + self.assertEqual(d.result, 42) + + def test_show(self): + # _QueryString hides the entered text when show is given. + d = self.open(_QueryString, show='*') + self.assertEqual(str(d.entry.cget('show')), '*') + + # --- Accept and cancel --- + + def test_return_accepts(self): + for query, value, expected in [ + (_QueryInteger, '42', 42), + (_QueryFloat, '1.5', 1.5), + (_QueryString, 'spam', 'spam'), + ]: + with self.subTest(query=query.__name__): + d = self.open(query) + self.enter(d, value) + self.assertEqual(d.result, expected) + self.assertFalse(d.winfo_exists()) # The dialog closed. + + def test_escape_cancels(self): + d = self.open(_QueryString) + self.enter(d, 'spam', '') + self.assertIsNone(d.result) + self.assertFalse(d.winfo_exists()) + + # --- Validation --- + + def test_invalid_value(self): + warnings = [] + d = self.open(_QueryInteger) + with swap_attr(messagebox, 'showwarning', + lambda *a, **k: warnings.append(a)): + self.enter(d, 'not a number') + self.assertIsNone(d.result) + self.assertTrue(d.winfo_exists()) # The dialog stays open. + self.assertTrue(warnings) + + def test_out_of_range(self): + warnings = [] + d = self.open(_QueryInteger, minvalue=10, maxvalue=20) + with swap_attr(messagebox, 'showwarning', + lambda *a, **k: warnings.append(a)): + self.enter(d, '5') # Below the minimum. + self.assertIsNone(d.result) + self.enter(d, '25') # Above the maximum. + self.assertIsNone(d.result) + self.assertTrue(d.winfo_exists()) + self.assertEqual(len(warnings), 2) + + def test_boundary_values_accepted(self): + # The min/max checks are inclusive: a value equal to a bound passes. + d = self.open(_QueryInteger, minvalue=10, maxvalue=20) + self.enter(d, '10') # Exactly the minimum. + self.assertEqual(d.result, 10) + self.assertFalse(d.winfo_exists()) + + d = self.open(_QueryInteger, minvalue=10, maxvalue=20) + self.enter(d, '20') # Exactly the maximum. + self.assertEqual(d.result, 20) + self.assertFalse(d.winfo_exists()) + + # --- Convenience ask* functions --- + + def run_ask(self, ask, value, **kw): + # Drive a modal ask* function: enter a value and accept it. + def accept(d): + d.entry.delete(0, 'end') + d.entry.insert(0, value) + d.ok() + with swap_attr(Dialog, 'wait_window', staticmethod(accept)): + return ask('Title', 'Prompt', parent=self.root, **kw) + + def test_ask_functions(self): + self.assertEqual(self.run_ask(askinteger, '42'), 42) + self.assertEqual(self.run_ask(askfloat, '1.5'), 1.5) + self.assertEqual(self.run_ask(askstring, 'spam'), 'spam') + + def test_ask_cancelled(self): + # A cancelled ask* returns None. + with swap_attr(Dialog, 'wait_window', staticmethod(lambda d: d.cancel())): + self.assertIsNone(askstring('Title', 'Prompt', parent=self.root)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index 453a4505a0a4da8..905c194af3ac474 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -1,8 +1,9 @@ import unittest import tkinter +from tkinter import TclError from test.support import requires from test.test_tkinter.support import setUpModule # noqa: F401 -from test.test_tkinter.support import AbstractTkTest +from test.test_tkinter.support import AbstractTkTest, requires_tk requires('gui') @@ -25,21 +26,598 @@ def test_debug(self): text.debug(olddebug) self.assertEqual(text.debug(), olddebug) + def test_sync(self): + text = self.text + # sync() returns None and brings line metrics up to date. + self.assertIsNone(text.sync()) + self.assertIs(text.pendingsync(), False) + + # sync(command) schedules a one-shot callback. + events = [] + text.sync(command=lambda: events.append('synced')) + text.update() + self.assertEqual(events, ['synced']) + + def test_index(self): + text = self.text + text.insert('1.0', 'Lorem ipsum\ndolor sit amet') + self.assertEqual(text.index('1.0'), '1.0') + self.assertEqual(text.index('1.end'), '1.11') + self.assertEqual(text.index('end'), '3.0') + self.assertEqual(text.index('2.5'), '2.5') + # Index past the end of a line is clamped to its end. + self.assertEqual(text.index('1.100'), '1.11') + self.assertRaises(TclError, text.index, 'invalid') + self.assertRaises(TypeError, text.index) + self.assertRaises(TypeError, text.index, '1.0', '2.5') + + def test_compare(self): + text = self.text + text.insert('1.0', 'Lorem ipsum\ndolor sit amet') + self.assertIs(text.compare('1.0', '<', '2.0'), True) + self.assertIs(text.compare('2.0', '<', '1.0'), False) + self.assertIs(text.compare('1.5', '==', '1.5'), True) + self.assertIs(text.compare('1.5', '!=', '1.5'), False) + self.assertIs(text.compare('2.0', '>=', '2.0'), True) + self.assertIs(text.compare('1.0', '<=', 'end'), True) + self.assertRaises(TclError, text.compare, '1.0', 'invalid', '2.0') + self.assertRaises(TypeError, text.compare, '1.0', '<') + self.assertRaises(TypeError, text.compare, '1.0', '<', '2.0', '3.0') + + def test_insert_get_delete(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + self.assertEqual(text.get('1.0', 'end'), 'Lorem ipsum\n') + self.assertEqual(text.get('1.0', '1.5'), 'Lorem') + self.assertEqual(text.get('1.6'), 'i') # single character + + # insert before an existing index + text.insert('1.0', '*** ') + self.assertEqual(text.get('1.0', 'end'), '*** Lorem ipsum\n') + + text.delete('1.0', '1.4') + self.assertEqual(text.get('1.0', 'end'), 'Lorem ipsum\n') + text.delete('1.5') # delete a single character + self.assertEqual(text.get('1.0', 'end'), 'Loremipsum\n') + self.assertRaises(TypeError, text.get) + self.assertRaises(TypeError, text.get, '1.0', '1.5', 'end') + self.assertRaises(TypeError, text.delete, '1.0', '1.5', 'end') + + def test_insert_with_tags(self): + text = self.text + text.insert('1.0', 'hello', 'a', ' ', ('a', 'b'), 'world', 'b') + self.assertEqual(text.get('1.0', 'end'), 'hello world\n') + self.assertEqual([str(i) for i in text.tag_ranges('a')], + ['1.0', '1.6']) + self.assertEqual([str(i) for i in text.tag_ranges('b')], + ['1.5', '1.11']) + + def test_replace(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + text.replace('1.0', '1.5', 'Hello') + self.assertEqual(text.get('1.0', 'end'), 'Hello ipsum\n') + text.replace('1.6', 'end - 1 char', 'world') + self.assertEqual(text.get('1.0', 'end'), 'Hello world\n') + self.assertRaises(TclError, text.replace, 'invalid', '1.5', 'x') + # The first index must not be after the second. + self.assertRaises(TclError, text.replace, '1.5', '1.0', 'x') + self.assertRaises(TypeError, text.replace, '1.0', '1.5') + + def test_mark(self): + text = self.text + text.insert('1.0', 'Lorem ipsum\ndolor sit amet') + # 'insert' and 'current' marks always exist. + self.assertIn('insert', text.mark_names()) + self.assertIn('current', text.mark_names()) + + text.mark_set('here', '1.5') + self.assertIn('here', text.mark_names()) + self.assertEqual(text.index('here'), '1.5') + text.mark_set('here', '2.3') + self.assertEqual(text.index('here'), '2.3') + + text.mark_unset('here') + self.assertNotIn('here', text.mark_names()) + + # Unsetting a non-existent mark is not an error. + text.mark_unset('nonexistent') + self.assertRaises(TclError, text.mark_set, 'm', 'invalid') + self.assertRaises(TypeError, text.mark_set, 'm') + self.assertRaises(TypeError, text.mark_set, 'm', '1.0', '2.0') + + def test_mark_gravity(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + text.mark_set('here', '1.5') + # The default gravity is right. + self.assertEqual(text.mark_gravity('here'), 'right') + text.mark_gravity('here', 'left') + self.assertEqual(text.mark_gravity('here'), 'left') + # With left gravity the mark stays before inserted text. + text.insert('here', 'XXX') + self.assertEqual(text.index('here'), '1.5') + # With right gravity the mark moves after inserted text. + text.mark_gravity('here', 'right') + text.insert('here', 'YYY') + self.assertEqual(text.index('here'), '1.8') + + self.assertRaises(TclError, text.mark_gravity, 'nonexistent') + self.assertRaises(TclError, text.mark_gravity, 'here', 'invalid') + self.assertRaises(TypeError, text.mark_gravity) + self.assertRaises(TypeError, text.mark_gravity, 'here', 'left', 'extra') + + def test_mark_next_previous(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + # Keep the builtin 'insert' and 'current' marks at 1.0 so they do + # not interfere with the queries below. + text.mark_set('insert', '1.0') + text.mark_set('current', '1.0') + text.mark_set('m1', '1.3') + text.mark_set('m2', '1.7') + # mark_next finds a mark at a position at or after the index, + # mark_previous finds one at a position strictly before it. + self.assertEqual(text.mark_next('1.1'), 'm1') + self.assertEqual(text.mark_next('1.3'), 'm1') + self.assertEqual(text.mark_next('1.4'), 'm2') + self.assertIsNone(text.mark_next('1.8')) + self.assertEqual(text.mark_previous('1.4'), 'm1') + self.assertEqual(text.mark_previous('1.7'), 'm1') + self.assertEqual(text.mark_previous('end'), 'm2') + self.assertIsNone(text.mark_previous('1.0')) + + self.assertRaises(TclError, text.mark_next, 'invalid') + self.assertRaises(TclError, text.mark_previous, 'invalid') + self.assertRaises(TypeError, text.mark_next) + self.assertRaises(TypeError, text.mark_previous) + self.assertRaises(TypeError, text.mark_next, '1.0', '2.0') + self.assertRaises(TypeError, text.mark_previous, '1.0', '2.0') + + def test_tag_add_remove_ranges(self): + text = self.text + text.insert('1.0', 'Lorem ipsum\ndolor sit amet') + self.assertEqual(text.tag_ranges('sel'), ()) + + text.tag_add('big', '1.0', '1.5') + self.assertEqual([str(i) for i in text.tag_ranges('big')], + ['1.0', '1.5']) + # Add a second, disjoint range. + text.tag_add('big', '2.0', '2.5') + self.assertEqual([str(i) for i in text.tag_ranges('big')], + ['1.0', '1.5', '2.0', '2.5']) + + text.tag_remove('big', '1.0', '1.3') + self.assertEqual([str(i) for i in text.tag_ranges('big')], + ['1.3', '1.5', '2.0', '2.5']) + + # tag_ranges of an undefined tag is empty, not an error. + self.assertEqual(text.tag_ranges('nonexistent'), ()) + self.assertRaises(TclError, text.tag_add, 'big', 'invalid') + self.assertRaises(TypeError, text.tag_add, 'big') + self.assertRaises(TclError, text.tag_remove, 'big', 'invalid') + self.assertRaises(TypeError, text.tag_remove, 'big', '1.0', '2.0', '3.0') + self.assertRaises(TypeError, text.tag_ranges, 'big', 'extra') + + def test_tag_names(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + text.tag_add('a', '1.0', '1.5') + text.tag_add('b', '1.3', '1.8') + # 'sel' always exists; order reflects priority (creation order). + self.assertEqual(set(text.tag_names()), {'sel', 'a', 'b'}) + # Tags applied to the character at the given index. + self.assertEqual(set(text.tag_names('1.4')), {'a', 'b'}) + self.assertEqual(set(text.tag_names('1.0')), {'a'}) + self.assertEqual(set(text.tag_names('1.10')), set()) + self.assertRaises(TclError, text.tag_names, 'invalid') + self.assertRaises(TypeError, text.tag_names, '1.0', '2.0') + + def test_tag_nextrange_prevrange(self): + text = self.text + text.insert('1.0', 'Lorem ipsum dolor') + text.tag_add('a', '1.0', '1.5') + text.tag_add('a', '1.12', '1.17') + + self.assertEqual([str(i) for i in text.tag_nextrange('a', '1.0')], + ['1.0', '1.5']) + self.assertEqual([str(i) for i in text.tag_nextrange('a', '1.5')], + ['1.12', '1.17']) + self.assertEqual(text.tag_nextrange('a', '1.17'), ()) + + self.assertEqual([str(i) for i in text.tag_prevrange('a', 'end')], + ['1.12', '1.17']) + self.assertEqual([str(i) for i in text.tag_prevrange('a', '1.12')], + ['1.0', '1.5']) + self.assertEqual(text.tag_prevrange('a', '1.0'), ()) + + # An undefined tag has no ranges, but the index must be valid. + self.assertEqual(text.tag_nextrange('nonexistent', '1.0'), ()) + self.assertRaises(TclError, text.tag_nextrange, 'a', 'invalid') + self.assertRaises(TclError, text.tag_prevrange, 'a', 'invalid') + self.assertRaises(TypeError, text.tag_nextrange, 'a') + self.assertRaises(TypeError, text.tag_prevrange, 'a') + self.assertRaises(TypeError, text.tag_nextrange, 'a', '1.0', '2.0', '3.0') + self.assertRaises(TypeError, text.tag_prevrange, 'a', '1.0', '2.0', '3.0') + + def test_tag_delete(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + text.tag_add('a', '1.0', '1.5') + text.tag_add('b', '1.6', '1.11') + self.assertEqual(set(text.tag_names()), {'sel', 'a', 'b'}) + text.tag_delete('a', 'b') + self.assertEqual(set(text.tag_names()), {'sel'}) + self.assertEqual(text.tag_ranges('a'), ()) + + def test_tag_raise_lower(self): + text = self.text + text.tag_configure('a', foreground='red') + text.tag_configure('b', foreground='green') + text.tag_configure('c', foreground='blue') + # Creation order is lowest-to-highest priority; tag_names lists + # them in increasing priority order. + self.assertEqual(text.tag_names(), ('sel', 'a', 'b', 'c')) + text.tag_raise('a') + self.assertEqual(text.tag_names(), ('sel', 'b', 'c', 'a')) + text.tag_lower('a') + self.assertEqual(text.tag_names(), ('a', 'sel', 'b', 'c')) + text.tag_raise('a', 'b') + self.assertEqual(text.tag_names(), ('sel', 'b', 'a', 'c')) + text.tag_lower('c', 'b') + self.assertEqual(text.tag_names(), ('sel', 'c', 'b', 'a')) + + self.assertRaises(TclError, text.tag_raise, 'nonexistent') + self.assertRaises(TclError, text.tag_lower, 'nonexistent') + self.assertRaises(TclError, text.tag_raise, 'a', 'nonexistent') + self.assertRaises(TypeError, text.tag_raise) + self.assertRaises(TypeError, text.tag_raise, 'a', 'b', 'c') + self.assertRaises(TypeError, text.tag_lower, 'a', 'b', 'c') + + def test_tag_cget_configure(self): + text = self.text + text.tag_configure('a', foreground='red', underline=True) + self.assertEqual(str(text.tag_cget('a', 'foreground')), 'red') + self.assertIn(text.tag_cget('a', 'underline'), (1, '1', True)) + # tag_cget normalizes the option name (no leading hyphen needed). + self.assertEqual(str(text.tag_cget('a', '-foreground')), 'red') + # configure() query returns the full option spec. + self.assertEqual(text.tag_configure('a', 'foreground')[-1], 'red') + text.tag_configure('a', foreground='blue') + self.assertEqual(str(text.tag_cget('a', 'foreground')), 'blue') + + self.assertRaises(TclError, text.tag_cget, 'nonexistent', + 'foreground') + self.assertRaises(TypeError, text.tag_cget, 'a') + self.assertRaises(TypeError, text.tag_cget, 'a', 'foreground', 'extra') + + def test_edit_modified(self): + text = self.text + self.assertEqual(text.edit_modified(), 0) + text.insert('1.0', 'spam') + self.assertEqual(text.edit_modified(), 1) + text.edit_modified(False) + self.assertEqual(text.edit_modified(), 0) + text.edit_modified(True) + self.assertEqual(text.edit_modified(), 1) + + def test_edit_undo_redo(self): + text = self.text + text.configure(undo=True) + text.insert('1.0', 'spam') + text.edit_separator() + text.insert('end', ' eggs') + self.assertEqual(text.get('1.0', 'end'), 'spam eggs\n') + + text.edit_undo() + self.assertEqual(text.get('1.0', 'end'), 'spam\n') + text.edit_redo() + self.assertEqual(text.get('1.0', 'end'), 'spam eggs\n') + + # An empty undo stack raises. + text.edit_reset() + self.assertRaises(TclError, text.edit_undo) + + def test_edit_canundo_canredo(self): + text = self.text + text.configure(undo=True) + + self.assertIs(text.edit_canundo(), False) + self.assertIs(text.edit_canredo(), False) + + text.insert('1.0', 'spam') + self.assertIs(text.edit_canundo(), True) + self.assertIs(text.edit_canredo(), False) + + text.edit_undo() + self.assertIs(text.edit_canundo(), False) + self.assertIs(text.edit_canredo(), True) + + text.edit_redo() + self.assertIs(text.edit_canundo(), True) + self.assertIs(text.edit_canredo(), False) + + text.edit_reset() + self.assertIs(text.edit_canundo(), False) + self.assertIs(text.edit_canredo(), False) + + def test_dump(self): + text = self.text + text.insert('1.0', 'hello') + text.tag_add('a', '1.0', '1.3') + text.mark_set('here', '1.2') + + dump = text.dump('1.0', 'end') + self.assertIsInstance(dump, list) + for item in dump: + self.assertIsInstance(item, tuple) + self.assertEqual(len(item), 3) + kinds = {item[0] for item in dump} + self.assertIn('text', kinds) + self.assertIn('tagon', kinds) + self.assertIn('mark', kinds) + + # Filtering by kind. + text_only = text.dump('1.0', 'end', text=True) + self.assertEqual({item[0] for item in text_only}, {'text'}) + self.assertEqual(''.join(item[1] for item in text_only), 'hello\n') + + # The command callback receives each triple and suppresses the result. + collected = [] + result = text.dump('1.0', 'end', text=True, + command=lambda *a: collected.append(a)) + self.assertIsNone(result) + self.assertEqual(''.join(key_value_index[1] + for key_value_index in collected), 'hello\n') + + self.assertRaises(TclError, text.dump, 'invalid') + self.assertRaises(TypeError, text.dump) + self.assertRaises(TypeError, text.dump, '1.0', 'end', None, 'extra') + + def test_image(self): + text = self.text + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + text.insert('1.0', 'AB') + name = text.image_create('1.1', image=image) + self.assertIsInstance(name, str) + self.assertIn(name, self.root.splitlist(text.image_names())) + self.assertEqual(str(text.image_cget(name, 'image')), str(image)) + + text.image_configure(name, align='top') + self.assertEqual(str(text.image_cget(name, 'align')), 'top') + + # An embedded image occupies a single index position. + self.assertEqual(text.index('end - 1 char'), '1.3') + + # Either a name or an image is required, and the index must be valid. + self.assertRaises(TclError, text.image_create, '1.0') + self.assertRaises(TclError, text.image_create, 'invalid', + image=image) + self.assertRaises(TypeError, text.image_cget, name) + self.assertRaises(TypeError, text.image_cget, name, 'image', 'extra') + + def test_window(self): + text = self.text + button = tkinter.Button(text, text='ok') + text.insert('1.0', 'AB') + text.window_create('1.1', window=button) + self.assertEqual(self.root.splitlist(text.window_names()), + (str(button),)) + self.assertEqual(text.window_cget('1.1', 'window'), str(button)) + + text.window_configure('1.1', padx=5) + self.assertEqual(text.window_cget('1.1', 'padx'), 5) + + # No embedded window at this index, and the index must be valid. + self.assertRaises(TclError, text.window_cget, '1.0', 'window') + self.assertRaises(TclError, text.window_cget, 'invalid', 'window') + self.assertRaises(TypeError, text.window_cget, '1.1') + self.assertRaises(TypeError, text.window_cget, '1.1', 'window', 'extra') + button.destroy() + + def test_tag_configure(self): + text = self.text + tag = 'a' + getint = text.tk.getint + getboolean = text.tk.getboolean + + # Color options. + for opt in ('foreground', 'background'): + text.tag_configure(tag, **{opt: '#ff0000'}) + self.assertEqual(str(text.tag_cget(tag, opt)), '#ff0000') + + # Stipple (bitmap) options. + for opt in ('bgstipple', 'fgstipple'): + text.tag_configure(tag, **{opt: 'gray50'}) + self.assertEqual(str(text.tag_cget(tag, opt)), 'gray50') + + # Enumerated options. + for opt, value in (('justify', 'center'), ('wrap', 'word'), + ('relief', 'raised'), ('tabstyle', 'wordprocessor'), + ('offset', '5')): + text.tag_configure(tag, **{opt: value}) + self.assertEqual(str(text.tag_cget(tag, 'justify')), 'center') + self.assertEqual(str(text.tag_cget(tag, 'wrap')), 'word') + self.assertEqual(str(text.tag_cget(tag, 'relief')), 'raised') + self.assertEqual(str(text.tag_cget(tag, 'tabstyle')), 'wordprocessor') + + # Boolean options. + for opt in ('underline', 'overstrike', 'elide'): + text.tag_configure(tag, **{opt: True}) + self.assertIs(getboolean(text.tag_cget(tag, opt)), True) + text.tag_configure(tag, **{opt: False}) + self.assertIs(getboolean(text.tag_cget(tag, opt)), False) + + # Screen-distance (pixel) options. + for opt in ('borderwidth', 'lmargin1', 'lmargin2', 'rmargin', + 'spacing1', 'spacing2', 'spacing3'): + text.tag_configure(tag, **{opt: 7}) + self.assertEqual(getint(text.tag_cget(tag, opt)), 7) + + # Other options. + text.tag_configure(tag, font='Helvetica 12') + self.assertEqual(str(text.tag_cget(tag, 'font')), 'Helvetica 12') + text.tag_configure(tag, tabs=(10.2, '1i')) + self.assertEqual([str(x) for x in text.tag_ranges('sel')], []) + + self.assertRaises(TclError, text.tag_cget, tag, 'spam') + + @requires_tk(8, 6, 6) + def test_tag_configure_colors(self): + # Tag color options added in Tk 8.6.6. + text = self.text + tag = 'a' + for opt in ('selectforeground', 'selectbackground', + 'lmargincolor', 'rmargincolor', + 'underlinefg', 'overstrikefg'): + text.tag_configure(tag, **{opt: '#00ff00'}) + self.assertEqual(str(text.tag_cget(tag, opt)), '#00ff00') + + def test_tag_configure_query(self): + text = self.text + tag = 'a' + # Querying all options returns a dict keyed by option name. + cnf = text.tag_configure(tag) + self.assertIsInstance(cnf, dict) + self.assertIn('foreground', cnf) + # The value is the full 5-tuple option specification. + self.assertEqual(len(cnf['foreground']), 5) + + # Querying a single option returns its specification. + spec = text.tag_configure(tag, 'foreground') + self.assertEqual(spec[0], 'foreground') + self.assertEqual(spec[-1], '') # unset by default + + # Setting via keyword arguments and via a dict are equivalent. + text.tag_configure(tag, foreground='red') + self.assertEqual(str(text.tag_configure(tag, 'foreground')[-1]), 'red') + text.tag_configure(tag, {'foreground': 'blue'}) + self.assertEqual(str(text.tag_cget(tag, 'foreground')), 'blue') + + # tag_config is an alias of tag_configure. + self.assertEqual(text.tag_config, text.tag_configure) + + def test_image_configure(self): + text = self.text + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + text.insert('1.0', 'AB') + name = text.image_create('1.1', image=image, name='img') + self.assertEqual(name, 'img') + + self.assertEqual(str(text.image_cget(name, 'name')), 'img') + self.assertEqual(str(text.image_cget(name, 'image')), str(image)) + for value in ('top', 'center', 'bottom', 'baseline'): + text.image_configure(name, align=value) + self.assertEqual(str(text.image_cget(name, 'align')), value) + text.image_configure(name, padx=3, pady=4) + self.assertEqual(text.tk.getint(text.image_cget(name, 'padx')), 3) + self.assertEqual(text.tk.getint(text.image_cget(name, 'pady')), 4) + + # Querying returns the full option set. + cnf = text.image_configure(name) + self.assertIsInstance(cnf, dict) + self.assertIn('align', cnf) + self.assertRaises(TclError, text.image_cget, name, 'spam') + + def test_window_configure(self): + text = self.text + button = tkinter.Button(text, text='ok') + text.insert('1.0', 'AB') + text.window_create('1.1', window=button) + + self.assertEqual(text.window_cget('1.1', 'window'), str(button)) + for value in ('top', 'center', 'bottom', 'baseline'): + text.window_configure('1.1', align=value) + self.assertEqual(str(text.window_cget('1.1', 'align')), value) + text.window_configure('1.1', padx=3, pady=4) + self.assertEqual(text.tk.getint(text.window_cget('1.1', 'padx')), 3) + self.assertEqual(text.tk.getint(text.window_cget('1.1', 'pady')), 4) + text.window_configure('1.1', stretch=True) + self.assertIs(text.tk.getboolean(text.window_cget('1.1', 'stretch')), + True) + + cnf = text.window_configure('1.1') + self.assertIsInstance(cnf, dict) + self.assertIn('stretch', cnf) + self.assertRaises(TclError, text.window_cget, '1.1', 'spam') + self.assertEqual(text.window_config, text.window_configure) + button.destroy() + + def test_peer(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + self.assertEqual(text.peer_names(), ()) + + text.peer_create('.peer1') + names = self.root.splitlist(text.tk.call('winfo', 'children', '.')) + self.assertIn('.peer1', [str(n) for n in names]) + self.assertEqual([str(p) for p in text.peer_names()], ['.peer1']) + # Peers share content. + self.assertEqual(text.tk.call('.peer1', 'get', '1.0', 'end'), + 'Lorem ipsum\n') + text.tk.call('destroy', '.peer1') + + def test_bbox(self): + text = self.text + text.insert('1.0', 'hello') + text.update() + bbox = text.bbox('1.0') + self.assertIsInstance(bbox, tuple) + self.assertEqual(len(bbox), 4) + for v in bbox: + self.assertIsInstance(v, int) + # A character that is not displayed has no bounding box. + self.assertIsNone(text.bbox('end')) + self.assertRaises(TclError, text.bbox, 'invalid') + self.assertRaises(TypeError, text.bbox) + self.assertRaises(TypeError, text.bbox, '1.0', '2.0') + + def test_dlineinfo(self): + text = self.text + text.insert('1.0', 'hello\nworld') + text.update() + info = text.dlineinfo('1.0') + self.assertIsInstance(info, tuple) + self.assertEqual(len(info), 5) + for v in info: + self.assertIsInstance(v, int) + self.assertRaises(TclError, text.dlineinfo, 'invalid') + self.assertRaises(TypeError, text.dlineinfo) + self.assertRaises(TypeError, text.dlineinfo, '1.0', '2.0') + + def test_see(self): + text = self.text + text.insert('1.0', '\n'.join('line %d' % i for i in range(200))) + text.update() + # Initially the last line is not visible. + self.assertIsNone(text.bbox('200.0')) + text.see('200.0') + text.update() + self.assertIsNotNone(text.bbox('200.0')) + self.assertRaises(TclError, text.see, 'invalid') + self.assertRaises(TypeError, text.see) + self.assertRaises(TypeError, text.see, '1.0', '2.0') + + # yview_pickplace is a deprecated way to make an index visible. + text.yview_pickplace('1.0') + text.update() + self.assertIsNotNone(text.bbox('1.0')) + def test_search(self): text = self.text # pattern and index are obligatory arguments. - self.assertRaises(tkinter.TclError, text.search, None, '1.0') - self.assertRaises(tkinter.TclError, text.search, 'a', None) - self.assertRaises(tkinter.TclError, text.search, None, None) + self.assertRaises(TclError, text.search, None, '1.0') + self.assertRaises(TclError, text.search, 'a', None) + self.assertRaises(TclError, text.search, None, None) # Invalid text index. - self.assertRaises(tkinter.TclError, text.search, '', 0) - self.assertRaises(tkinter.TclError, text.search, '', '') - self.assertRaises(tkinter.TclError, text.search, '', 'invalid') - self.assertRaises(tkinter.TclError, text.search, '', '1.0', 0) - self.assertRaises(tkinter.TclError, text.search, '', '1.0', '') - self.assertRaises(tkinter.TclError, text.search, '', '1.0', 'invalid') + self.assertRaises(TclError, text.search, '', 0) + self.assertRaises(TclError, text.search, '', '') + self.assertRaises(TclError, text.search, '', 'invalid') + self.assertRaises(TclError, text.search, '', '1.0', 0) + self.assertRaises(TclError, text.search, '', '1.0', '') + self.assertRaises(TclError, text.search, '', '1.0', 'invalid') text.insert('1.0', 'This is a test. This is only a test.\n' @@ -88,20 +666,20 @@ def test_search_all(self): text = self.text # pattern and index are obligatory arguments. - self.assertRaises(tkinter.TclError, text.search_all, None, '1.0') - self.assertRaises(tkinter.TclError, text.search_all, 'a', None) - self.assertRaises(tkinter.TclError, text.search_all, None, None) + self.assertRaises(TclError, text.search_all, None, '1.0') + self.assertRaises(TclError, text.search_all, 'a', None) + self.assertRaises(TclError, text.search_all, None, None) # Keyword-only arguments self.assertRaises(TypeError, text.search_all, 'a', '1.0', 'end', None) # Invalid text index. - self.assertRaises(tkinter.TclError, text.search_all, '', 0) - self.assertRaises(tkinter.TclError, text.search_all, '', '') - self.assertRaises(tkinter.TclError, text.search_all, '', 'invalid') - self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 0) - self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', '') - self.assertRaises(tkinter.TclError, text.search_all, '', '1.0', 'invalid') + self.assertRaises(TclError, text.search_all, '', 0) + self.assertRaises(TclError, text.search_all, '', '') + self.assertRaises(TclError, text.search_all, '', 'invalid') + self.assertRaises(TclError, text.search_all, '', '1.0', 0) + self.assertRaises(TclError, text.search_all, '', '1.0', '') + self.assertRaises(TclError, text.search_all, '', '1.0', 'invalid') def eq(res, expected): self.assertIsInstance(res, tuple) @@ -181,8 +759,8 @@ def test_count(self): self.assertEqual(text.count('1.0', 'end'), (124,)) self.assertEqual(text.count('1.0', 'end', 'indices', return_ints=True), 124) self.assertEqual(text.count('1.0', 'end', 'indices'), (124,)) - self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', 'spam') - self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', '-lines') + self.assertRaises(TclError, text.count, '1.0', 'end', 'spam') + self.assertRaises(TclError, text.count, '1.0', 'end', '-lines') self.assertIsInstance(text.count('1.3', '1.5', 'ypixels', return_ints=True), int) self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), tuple) diff --git a/Lib/test/test_tkinter/test_variables.py b/Lib/test/test_tkinter/test_variables.py index 8733095ffb65f40..5ee4e9467dd0cba 100644 --- a/Lib/test/test_tkinter/test_variables.py +++ b/Lib/test/test_tkinter/test_variables.py @@ -111,6 +111,7 @@ def test_initialize(self): self.assertFalse(v.side_effect) v.set("value") self.assertTrue(v.side_effect) + self.assertEqual(Variable.initialize, Variable.set) def test_trace_old(self): if tcl_version >= (9, 0): @@ -118,6 +119,7 @@ def test_trace_old(self): # Old interface v = Variable(self.root) vname = str(v) + self.assertEqual(v.trace, v.trace_variable) trace = [] def read_tracer(*args): trace.append(('read',) + args) @@ -328,6 +330,7 @@ def test_set(self): self.assertEqual(self.root.globalgetvar("name"), false) v.set("on") self.assertEqual(self.root.globalgetvar("name"), true) + self.assertEqual(BooleanVar.initialize, BooleanVar.set) def test_invalid_value_domain(self): false = 0 if self.root.wantobjects() else "0" diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index 1c400e970eb02da..4b7a6acfad0b922 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -208,6 +208,23 @@ def test_configure_default(self): widget = self.create() self.checkEnumParam(widget, 'default', 'active', 'disabled', 'normal') + def test_invoke(self): + success = [] + widget = self.create(command=lambda: success.append(1)) + widget.pack() + widget.invoke() + self.assertEqual(success, [1]) + # invoke does nothing for a disabled button. + widget.configure(state='disabled') + widget.invoke() + self.assertEqual(success, [1]) + + def test_flash(self): + widget = self.create() + widget.pack() + widget.update_idletasks() + widget.flash() # No exception. + @add_configure_tests(StandardOptionsTests) class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): @@ -263,6 +280,39 @@ def test_same_name(self): b2.deselect() self.assertEqual(v.get(), 0) + def test_invoke(self): + success = [] + v = tkinter.IntVar(self.root) + widget = self.create(variable=v, onvalue=1, offvalue=0, + command=lambda: success.append(v.get())) + widget.pack() + widget.invoke() + self.assertEqual(v.get(), 1) + self.assertEqual(success, [1]) + widget.invoke() + self.assertEqual(v.get(), 0) + self.assertEqual(success, [1, 0]) + # A disabled checkbutton is not toggled and its command is not called. + widget.configure(state='disabled') + widget.invoke() + self.assertEqual(v.get(), 0) + self.assertEqual(success, [1, 0]) + + def test_toggle(self): + v = tkinter.IntVar(self.root) + widget = self.create(variable=v, onvalue=1, offvalue=0) + self.assertEqual(v.get(), 0) + widget.toggle() + self.assertEqual(v.get(), 1) + widget.toggle() + self.assertEqual(v.get(), 0) + + def test_flash(self): + widget = self.create() + widget.pack() + widget.update_idletasks() + widget.flash() # No exception. + @add_configure_tests(StandardOptionsTests) class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( @@ -285,6 +335,28 @@ def test_configure_value(self): widget = self.create() self.checkParams(widget, 'value', 1, 2.3, '', 'any string') + def test_invoke(self): + success = [] + v = tkinter.StringVar(self.root) + widget = self.create(variable=v, value='on', + command=lambda: success.append(v.get())) + widget.pack() + widget.invoke() + self.assertEqual(v.get(), 'on') + self.assertEqual(success, ['on']) + # invoke does nothing for a disabled radiobutton. + v.set('') + widget.configure(state='disabled') + widget.invoke() + self.assertEqual(v.get(), '') + self.assertEqual(success, ['on']) + + def test_flash(self): + widget = self.create() + widget.pack() + widget.update_idletasks() + widget.flash() # No exception. + @add_configure_tests(StandardOptionsTests) class MenubuttonTest(AbstractLabelTest, unittest.TestCase): @@ -365,9 +437,22 @@ class OptionMenuTest(MenubuttonTest, unittest.TestCase): def create(self, default='b', values=('a', 'b', 'c'), **kwargs): return tkinter.OptionMenu(self.root, None, default, *values, **kwargs) + def test_kwargs(self): + # Menubutton options can be passed at construction (gh-101284). + widget = tkinter.OptionMenu(self.root, None, 'b', + width=10, direction='right') + self.assertEqual(int(widget['width']), 10) + self.assertEqual(str(widget['direction']), 'right') + # They override OptionMenu's own appearance defaults, + widget = tkinter.OptionMenu(self.root, None, 'b', relief='flat') + self.assertEqual(str(widget['relief']), 'flat') + # which otherwise keep their historical values. + widget = tkinter.OptionMenu(self.root, None, 'b') + self.assertEqual(str(widget['relief']), 'raised') + def test_bad_kwarg(self): - with self.assertRaisesRegex(TclError, r"^unknown option -image$"): - tkinter.OptionMenu(self.root, None, 'b', image='') + with self.assertRaisesRegex(TclError, r'^unknown option "-spam"$'): + tkinter.OptionMenu(self.root, None, 'b', spam='') def test_specify_name(self): widget = tkinter.OptionMenu(self.root, None, ':)', name="option_menu") @@ -476,6 +561,66 @@ def test_selection_methods(self): self.assertEqual(widget.selection_get(), '12345') widget.selection_adjust(0) + def test_delete(self): + widget = self.create() + widget.insert(0, 'abcdef') + widget.delete(1, 3) + self.assertEqual(widget.get(), 'adef') + widget.delete(1) + self.assertEqual(widget.get(), 'aef') + widget.delete(0, 'end') + self.assertEqual(widget.get(), '') + self.assertRaisesRegex(TclError, r'bad (entry|spinbox) index "xyz"', + widget.delete, 'xyz') + self.assertRaises(TypeError, widget.delete) + + def test_icursor(self): + widget = self.create() + widget.insert(0, 'abcdef') + widget.icursor(3) + widget.insert('insert', 'XYZ') + self.assertEqual(widget.get(), 'abcXYZdef') + self.assertRaisesRegex(TclError, r'bad (entry|spinbox) index "xyz"', + widget.icursor, 'xyz') + self.assertRaises(TypeError, widget.icursor) + + def test_select_aliases(self): + # The select_* methods are aliases of the selection_* methods. + widget = self.create() + widget.insert(0, '12345') + self.assertFalse(widget.select_present()) + widget.select_range(0, 'end') + self.assertTrue(widget.select_present()) + self.assertEqual(widget.selection_get(), '12345') + widget.select_from(1) + widget.select_to(3) + self.assertEqual(widget.selection_get(), '23') + widget.select_adjust(4) + self.assertEqual(widget.selection_get(), '234') + widget.select_clear() + self.assertFalse(widget.select_present()) + self.assertRaisesRegex(TclError, 'bad entry index "xyz"', + widget.select_range, 'xyz', 'end') + + def test_validate(self): + calls = [] + def validatecommand(value): + calls.append(value) + return value.isdigit() + # validate='none' means validation is never triggered automatically, + # so validate() exercises the forced evaluation. + widget = self.create(validate='none', + validatecommand=(self.root.register(validatecommand), '%P')) + widget.insert(0, '123') + result = widget.validate() + self.assertIs(result, True) + self.assertEqual(calls, ['123']) + widget.delete(0, 'end') + widget.insert(0, 'abc') + calls.clear() + self.assertIs(widget.validate(), False) + self.assertEqual(calls, ['abc']) + @add_configure_tests(StandardOptionsTests) class SpinboxTest(EntryTest, unittest.TestCase): @@ -624,6 +769,38 @@ def test_selection_element(self): widget.selection_element("buttondown") self.assertEqual(widget.selection_element(), "buttondown") + # Spinbox has no select_* aliases, unlike Entry. + test_select_aliases = None + + def test_invoke(self): + widget = self.create(from_=0, to=10) + widget.delete(0, 'end') + widget.insert(0, '5') + widget.invoke('buttonup') + self.assertEqual(widget.get(), '6') + widget.invoke('buttondown') + self.assertEqual(widget.get(), '5') + self.assertRaisesRegex(TclError, 'bad element "spam"', + widget.invoke, 'spam') + + def test_identify(self): + widget = self.create() + widget.pack() + widget.update_idletasks() + # The empty string is returned for a point over no element. + self.assertIn(widget.identify(5, 5), + ('entry', 'buttonup', 'buttondown', 'none', '')) + self.assertRaises(TclError, widget.identify, 'a', 'b') + + def test_scan(self): + widget = self.create() + widget.insert(0, 'a' * 100) + widget.pack() + widget.update_idletasks() + self.assertEqual(widget.scan_mark(10), ()) + self.assertEqual(widget.scan_dragto(0), ()) + self.assertRaises(TypeError, widget.scan_mark) + @add_configure_tests(StandardOptionsTests) class TextTest(AbstractWidgetTest, unittest.TestCase): @@ -978,6 +1155,122 @@ def test_create_polygon(self): self._test_option_smooth(c, lambda **kwargs: c.create_polygon(20, 30, 40, 50, 60, 10, **kwargs)) + def test_create_arc(self): + c = self.create() + i = c.create_arc(10, 20, 30, 40) + self.assertEqual(c.coords(i), [10.0, 20.0, 30.0, 40.0]) + self.assertEqual(c.itemcget(i, 'style'), 'pieslice') + self.assertEqual(float(c.itemcget(i, 'start')), 0.0) + self.assertEqual(float(c.itemcget(i, 'extent')), 90.0) + + for style in 'pieslice', 'chord', 'arc': + i = c.create_arc(10, 20, 30, 40, style=style) + self.assertEqual(c.itemcget(i, 'style'), style) + self.assertRaises(TclError, c.create_arc, 10, 20, 30, 40, style='spam') + + i = c.create_arc(10, 20, 30, 40, start=45, extent=120, + outline='red', fill='blue', width=3) + self.assertEqual(float(c.itemcget(i, 'start')), 45.0) + self.assertEqual(float(c.itemcget(i, 'extent')), 120.0) + self.assertEqual(str(c.itemcget(i, 'outline')), 'red') + self.assertEqual(str(c.itemcget(i, 'fill')), 'blue') + self.assertEqual(float(c.itemcget(i, 'width')), 3.0) + self.assertRaises(TclError, c.create_arc, 10, 20, 30, 40, extent='spam') + + def test_create_oval(self): + c = self.create() + i = c.create_oval(10, 20, 30, 40) + self.assertEqual(c.coords(i), [10.0, 20.0, 30.0, 40.0]) + self.assertEqual(c.bbox(i), (9, 19, 31, 41)) + self.assertEqual(c.itemcget(i, 'stipple'), '') + + i = c.create_oval(10, 20, 30, 40, fill='red', outline='blue', width=2) + self.assertEqual(str(c.itemcget(i, 'fill')), 'red') + self.assertEqual(str(c.itemcget(i, 'outline')), 'blue') + self.assertEqual(float(c.itemcget(i, 'width')), 2.0) + self.assertRaises(TclError, c.create_oval, 10, 20, 30, 40, width='spam') + + def test_create_bitmap(self): + c = self.create() + i = c.create_bitmap(10, 20, bitmap='gray50') + self.assertEqual(c.coords(i), [10.0, 20.0]) + self.assertEqual(c.itemcget(i, 'bitmap'), 'gray50') + self.assertEqual(c.itemcget(i, 'anchor'), 'center') + + for anchor in 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center': + i = c.create_bitmap(10, 20, bitmap='gray50', anchor=anchor) + self.assertEqual(c.itemcget(i, 'anchor'), anchor) + self.assertRaises(TclError, c.create_bitmap, 10, 20, + bitmap='gray50', anchor='spam') + + i = c.create_bitmap(10, 20, bitmap='gray50', + foreground='red', background='blue') + self.assertEqual(str(c.itemcget(i, 'foreground')), 'red') + self.assertEqual(str(c.itemcget(i, 'background')), 'blue') + if c._windowingsystem != 'aqua': + # Aqua resolves bitmaps lazily and does not report a bad name here. + self.assertRaises(TclError, c.create_bitmap, 10, 20, bitmap='spam') + + def test_create_image(self): + c = self.create() + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + i = c.create_image(10, 20, image=image) + self.assertEqual(c.coords(i), [10.0, 20.0]) + self.assertEqual(str(c.itemcget(i, 'image')), str(image)) + self.assertEqual(c.itemcget(i, 'anchor'), 'center') + + for anchor in 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center': + i = c.create_image(10, 20, image=image, anchor=anchor) + self.assertEqual(c.itemcget(i, 'anchor'), anchor) + self.assertRaises(TclError, c.create_image, 10, 20, + image=image, anchor='spam') + + def test_create_text(self): + c = self.create() + i = c.create_text(10, 20, text='Hello') + self.assertEqual(c.coords(i), [10.0, 20.0]) + self.assertEqual(c.itemcget(i, 'text'), 'Hello') + self.assertEqual(c.itemcget(i, 'anchor'), 'center') + self.assertEqual(c.itemcget(i, 'justify'), 'left') + + for justify in 'left', 'right', 'center': + i = c.create_text(10, 20, text='Hello', justify=justify) + self.assertEqual(c.itemcget(i, 'justify'), justify) + self.assertRaises(TclError, c.create_text, 10, 20, justify='spam') + + for anchor in 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center': + i = c.create_text(10, 20, text='Hello', anchor=anchor) + self.assertEqual(c.itemcget(i, 'anchor'), anchor) + self.assertRaises(TclError, c.create_text, 10, 20, anchor='spam') + + i = c.create_text(10, 20, text='Hello', fill='red', angle=45, + font='TkFixedFont') + self.assertEqual(str(c.itemcget(i, 'fill')), 'red') + self.assertEqual(float(c.itemcget(i, 'angle')), 45.0) + self.assertEqual(str(c.itemcget(i, 'font')), 'TkFixedFont') + self.assertRaises(TclError, c.create_text, 10, 20, angle='spam') + + def test_create_window(self): + c = self.create() + button = tkinter.Button(c, text='ok') + i = c.create_window(10, 20, window=button) + self.assertEqual(c.coords(i), [10.0, 20.0]) + self.assertEqual(c.itemcget(i, 'window'), str(button)) + self.assertEqual(c.itemcget(i, 'anchor'), 'center') + + for anchor in 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center': + i = c.create_window(10, 20, window=tkinter.Button(c), anchor=anchor) + self.assertEqual(c.itemcget(i, 'anchor'), anchor) + self.assertRaises(TclError, c.create_window, 10, 20, + window=button, anchor='spam') + + i = c.create_window(10, 20, window=tkinter.Button(c), + width=30, height=40) + self.assertEqual(int(c.itemcget(i, 'width')), 30) + self.assertEqual(int(c.itemcget(i, 'height')), 40) + self.assertRaises(TclError, c.create_window, 10, 20, + window=button, width='spam') + def test_coords(self): c = self.create() i = c.create_line(20, 30, 40, 50, 60, 10, tags='x') @@ -1038,6 +1331,419 @@ def test_moveto(self): self.assertEqual(x2_2 - x1_2, x2_3 - x1_3) self.assertEqual(y2_2 - y1_2, y2_3 - y1_3) + def test_create_items(self): + c = self.create() + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + button = tkinter.Button(c, text='ok') + items = { + 'arc': c.create_arc(10, 20, 30, 40), + 'bitmap': c.create_bitmap(10, 20, bitmap='gray50'), + 'image': c.create_image(10, 20, image=image), + 'line': c.create_line(10, 20, 30, 40), + 'oval': c.create_oval(10, 20, 30, 40), + 'polygon': c.create_polygon(10, 20, 30, 40, 50, 20), + 'rectangle': c.create_rectangle(10, 20, 30, 40), + 'text': c.create_text(10, 20, text='hello'), + 'window': c.create_window(10, 20, window=button), + } + for itemtype, item in items.items(): + self.assertIsInstance(item, int) + self.assertEqual(c.type(item), itemtype) + # All items are reported by find_all in creation (stacking) order. + self.assertEqual(c.find_all(), tuple(sorted(items.values()))) + + # No coordinates at all is an IndexError; a bad number is a TclError. + self.assertRaises(IndexError, c.create_arc) + self.assertRaises(TclError, c.create_arc, 1, 2, 3) + self.assertRaises(TclError, c.create_oval, 1, 2) + + def test_type(self): + c = self.create() + rect = c.create_rectangle(10, 20, 30, 40) + self.assertEqual(c.type(rect), 'rectangle') + # An unmatched tag or id is not an error. + self.assertIsNone(c.type('nonexistent')) + self.assertIsNone(c.type(9999)) + self.assertRaises(TypeError, c.type) + self.assertRaises(TypeError, c.type, rect, 'extra') + + def test_bbox(self): + c = self.create() + rect = c.create_rectangle(10, 10, 30, 30) + bbox = c.bbox(rect) + self.assertIsInstance(bbox, tuple) + self.assertEqual(len(bbox), 4) + for v in bbox: + self.assertIsInstance(v, int) + # The bounding box encloses the item (with a small margin). + self.assertEqual(bbox, (9, 9, 31, 31)) + # bbox over several items is their union. + rect2 = c.create_rectangle(50, 50, 70, 70) + self.assertEqual(c.bbox(rect, rect2), (9, 9, 71, 71)) + # An unmatched tag has no bounding box. + self.assertIsNone(c.bbox('nonexistent')) + # At least one tag or id is required. + self.assertRaises(TclError, c.bbox) + + def test_canvasx_canvasy(self): + c = self.create(borderwidth=0, highlightthickness=0) + self.assertEqual(c.canvasx(0), 0.0) + self.assertEqual(c.canvasx(10), 10.0) + self.assertIsInstance(c.canvasx(10), float) + # gridspacing rounds to the nearest multiple. + self.assertEqual(c.canvasx(13, 5), 15.0) + self.assertEqual(c.canvasy(0), 0.0) + self.assertEqual(c.canvasy(10), 10.0) + self.assertRaises(TypeError, c.canvasx) + self.assertRaises(TypeError, c.canvasx, 0, 5, 1) + self.assertRaises(TypeError, c.canvasy) + self.assertRaises(TypeError, c.canvasy, 0, 5, 1) + self.assertRaises(TclError, c.canvasx, 'spam') + self.assertRaises(TclError, c.canvasy, 'spam') + + def test_move(self): + c = self.create() + rect = c.create_rectangle(10, 10, 30, 30) + c.move(rect, 5, 7) + self.assertEqual(c.coords(rect), [15.0, 17.0, 35.0, 37.0]) + c.move(rect, -5, -7) + self.assertEqual(c.coords(rect), [10.0, 10.0, 30.0, 30.0]) + # move() takes variable arguments; bad or missing values reach Tcl. + self.assertRaises(TclError, c.move, rect, 'spam', 0) + self.assertRaises(TclError, c.move, rect) + + def test_scale(self): + c = self.create() + rect = c.create_rectangle(10, 10, 30, 30) + c.scale(rect, 0, 0, 2, 2) + self.assertEqual(c.coords(rect), [20.0, 20.0, 60.0, 60.0]) + c.scale(rect, 0, 0, 0.5, 0.5) + self.assertEqual(c.coords(rect), [10.0, 10.0, 30.0, 30.0]) + self.assertRaises(TclError, c.scale, rect, 0, 0, 'spam', 2) + self.assertRaises(TclError, c.scale, rect, 0, 0) # missing factors + + @requires_tk(8, 6) + def test_rchars(self): + c = self.create() + # On a line item, rchars replaces a range of the coordinate list. + line = c.create_line(0, 0, 10, 10, 20, 0) + c.rchars(line, 2, 5, (30, 30, 40, 40)) + self.assertEqual(c.coords(line), [0.0, 0.0, 30.0, 30.0, 40.0, 40.0]) + # On a text item, rchars replaces a range of characters. + text = c.create_text(10, 10, text='hello') + c.rchars(text, 0, 2, 'HE') + self.assertEqual(c.itemcget(text, 'text'), 'HElo') + self.assertRaises(TclError, c.rchars) + + @requires_tk(9, 0) + def test_rotate(self): + c = self.create() + line = c.create_line(10, 0, 20, 0) + # The canvas y-axis points down, so an anticlockwise rotation about + # the origin maps (x, y) to (y, -x). + c.rotate(line, 0, 0, 90) + for got, expected in zip(c.coords(line), [0, -10, 0, -20]): + self.assertAlmostEqual(got, expected, places=3) + # A negative angle rotates clockwise, restoring the original position. + c.rotate(line, 0, 0, -90) + for got, expected in zip(c.coords(line), [10, 0, 20, 0]): + self.assertAlmostEqual(got, expected, places=3) + self.assertRaises(TclError, c.rotate, line, 0, 0, 'spam') + + def test_delete(self): + c = self.create() + r1 = c.create_rectangle(10, 10, 30, 30) + r2 = c.create_rectangle(50, 50, 70, 70) + r3 = c.create_rectangle(90, 90, 110, 110) + self.assertEqual(c.find_all(), (r1, r2, r3)) + c.delete(r2) + self.assertEqual(c.find_all(), (r1, r3)) + # Deleting a non-existent item is not an error. + c.delete(9999) + c.delete('all') + self.assertEqual(c.find_all(), ()) + + def test_find(self): + c = self.create() + r1 = c.create_rectangle(10, 10, 30, 30) + r2 = c.create_rectangle(50, 50, 70, 70) + r3 = c.create_rectangle(100, 100, 120, 120) + + self.assertEqual(c.find_all(), (r1, r2, r3)) + # find_above/find_below return the single adjacent item. + self.assertEqual(c.find_above(r1), (r2,)) + self.assertEqual(c.find_below(r3), (r2,)) + self.assertEqual(c.find_above(r3), ()) # nothing above the top item + self.assertEqual(c.find_withtag(r2), (r2,)) + self.assertEqual(c.find_closest(60, 60), (r2,)) + self.assertEqual(c.find_enclosed(0, 0, 80, 80), (r1, r2)) + self.assertEqual(c.find_overlapping(0, 0, 20, 20), (r1,)) + # An unmatched query returns an empty tuple. + self.assertEqual(c.find_withtag('nonexistent'), ()) + for result in (c.find_all(), c.find_withtag(r1)): + self.assertIsInstance(result, tuple) + + self.assertRaises(TclError, c.find_closest, 'spam', 0) + self.assertRaises(TclError, c.find_enclosed, 0, 0, 'spam', 0) + self.assertRaises(TclError, c.find_overlapping, 0, 0, 'spam', 0) + self.assertRaises(TypeError, c.find_withtag) + self.assertRaises(TypeError, c.find_withtag, r1, 'extra') + self.assertRaises(TypeError, c.find_above) + self.assertRaises(TypeError, c.find_above, r1, 'extra') + self.assertRaises(TypeError, c.find_below) + self.assertRaises(TypeError, c.find_closest) + self.assertRaises(TypeError, c.find_closest, 0, 0, 1, 2, 3) + self.assertRaises(TypeError, c.find_enclosed, 0, 0, 1) + self.assertRaises(TypeError, c.find_enclosed, 0, 0, 1, 2, 3) + + def test_addtag_gettags_dtag(self): + c = self.create() + r1 = c.create_rectangle(10, 10, 30, 30) + r2 = c.create_rectangle(50, 50, 70, 70) + + self.assertEqual(c.gettags(r1), ()) + c.addtag_withtag('spam', r1) + self.assertEqual(c.gettags(r1), ('spam',)) + self.assertEqual(c.find_withtag('spam'), (r1,)) + + c.addtag_all('all') + self.assertIn('all', c.gettags(r1)) + self.assertIn('all', c.gettags(r2)) + + c.addtag_above('above1', r1) + self.assertIn('above1', c.gettags(r2)) + c.addtag_below('below2', r2) + self.assertIn('below2', c.gettags(r1)) + + c.addtag_enclosed('enc', 0, 0, 40, 40) + self.assertEqual(c.find_withtag('enc'), (r1,)) + c.addtag_overlapping('ov', 0, 0, 20, 20) + self.assertEqual(c.find_withtag('ov'), (r1,)) + c.addtag_closest('close', 60, 60) + self.assertEqual(c.find_withtag('close'), (r2,)) + + # gettags of an unmatched tag is empty. + self.assertEqual(c.gettags('nonexistent'), ()) + + # dtag removes a tag from an item. + c.dtag(r1, 'spam') + self.assertNotIn('spam', c.gettags(r1)) + + self.assertRaises(TypeError, c.addtag_withtag, 'tag') + self.assertRaises(TypeError, c.addtag_withtag, 'tag', r1, 'extra') + self.assertRaises(TypeError, c.addtag_all) + self.assertRaises(TypeError, c.addtag_enclosed, 'tag', 0, 0, 1) + self.assertRaises(TypeError, c.addtag_enclosed, 'tag', 0, 0, 1, 2, 3) + self.assertRaises(TclError, c.addtag_closest, 'tag', 'spam', 0) + self.assertRaises(TclError, c.addtag_enclosed, 'tag', 0, 0, 'spam', 0) + self.assertRaises(TclError, c.gettags) # needs an item + + def test_itemconfigure(self): + c = self.create() + rect = c.create_rectangle(10, 10, 30, 30) + c.itemconfigure(rect, fill='red', width=3) + self.assertEqual(str(c.itemcget(rect, 'fill')), 'red') + self.assertEqual(float(c.itemcget(rect, 'width')), 3.0) + + # Querying all options returns a dict; a single one returns its spec. + cnf = c.itemconfigure(rect) + self.assertIsInstance(cnf, dict) + self.assertIn('fill', cnf) + self.assertEqual(c.itemconfigure(rect, 'fill')[-1], 'red') + + # itemconfig is an alias of itemconfigure. + self.assertEqual(c.itemconfig, c.itemconfigure) + + self.assertRaises(TclError, c.itemcget, rect, 'badoption') + self.assertRaises(TclError, c.itemconfigure, rect, badoption='x') + self.assertRaises(TypeError, c.itemcget, rect) + self.assertRaises(TypeError, c.itemcget, rect, 'fill', 'extra') + + def test_tag_raise_lower(self): + c = self.create() + r1 = c.create_rectangle(10, 10, 30, 30) + r2 = c.create_rectangle(50, 50, 70, 70) + r3 = c.create_rectangle(90, 90, 110, 110) + self.assertEqual(c.find_all(), (r1, r2, r3)) + + c.tag_raise(r1) + self.assertEqual(c.find_all(), (r2, r3, r1)) + c.tag_lower(r1) + self.assertEqual(c.find_all(), (r1, r2, r3)) + # Raise above / lower below a specific item. + c.tag_raise(r1, r2) + self.assertEqual(c.find_all(), (r2, r1, r3)) + c.tag_lower(r3, r2) + self.assertEqual(c.find_all(), (r3, r2, r1)) + + # lower/lift are aliases of tag_lower/tag_raise. + self.assertEqual(c.lower, c.tag_lower) + self.assertEqual(c.lift, c.tag_raise) + + # raise/lower need at least an item; an unmatched tag is not an error. + self.assertRaises(TclError, c.tag_raise) + self.assertRaises(TclError, c.tag_lower) + self.assertIsNone(c.tag_raise('nonexistent')) + + def test_text_item(self): + c = self.create() + item = c.create_text(10, 10, text='Hello') + self.assertEqual(c.index(item, 'end'), 5) + self.assertIsInstance(c.index(item, 'end'), int) + + c.insert(item, 'end', ' world') + self.assertEqual(c.itemcget(item, 'text'), 'Hello world') + self.assertEqual(c.index(item, 'end'), 11) + c.insert(item, 0, '>> ') + self.assertEqual(c.itemcget(item, 'text'), '>> Hello world') + + c.dchars(item, 0, 2) + self.assertEqual(c.itemcget(item, 'text'), 'Hello world') + c.icursor(item, 3) + + # index requires an indexable (text) item and a valid index. + self.assertRaises(TclError, c.index, item, 'badspec') + self.assertRaises(TclError, c.index, item) # missing index + self.assertRaises(TclError, c.dchars, item, 'badspec', 'end') + rect = c.create_rectangle(10, 10, 30, 30) + self.assertRaises(TclError, c.index, rect, 'end') + + def test_select(self): + c = self.create() + item = c.create_text(10, 10, text='Hello world') + self.assertIsNone(c.select_item()) + + c.select_from(item, 0) + c.select_to(item, 4) + self.assertEqual(int(c.select_item()), item) + c.select_adjust(item, 6) + + c.select_clear() + self.assertIsNone(c.select_item()) + self.assertRaises(TypeError, c.select_from, item) + self.assertRaises(TypeError, c.select_from, item, 0, 'extra') + # A bad index reaches Tcl; selection applies only to text items. + self.assertRaises(TclError, c.select_from, item, 'badspec') + rect = c.create_rectangle(10, 10, 30, 30) + self.assertRaises(TclError, c.select_from, rect, 0) + + def test_focus(self): + c = self.create() + item = c.create_text(10, 10, text='Hello') + # Only text items can take the focus. + c.focus(item) + self.assertEqual(int(c.focus()), item) + c.focus('') + self.assertIn(c.focus(), ('', None)) + + def test_scan(self): + c = self.create() + c.create_rectangle(10, 10, 300, 300) + c.scan_mark(0, 0) + c.scan_dragto(5, 5) # default gain=10 + c.scan_dragto(5, 5, 1) + self.assertRaises(TypeError, c.scan_mark) + self.assertRaises(TypeError, c.scan_mark, 0, 0, 0) + self.assertRaises(TclError, c.scan_mark, 'spam', 0) + + def test_postscript(self): + c = self.create() + c.create_rectangle(10, 10, 30, 30, fill='black') + ps = c.postscript() + self.assertIsInstance(ps, str) + self.assertStartsWith(ps, '%!PS-Adobe') + self.assertRaises(TclError, c.postscript, badoption='spam') + + def assertCommandExist(self, widget, funcid): + self.assertEqual( + widget.tk.splitlist(widget.tk.call('info', 'commands', funcid)), + (funcid,)) + + def assertCommandNotExist(self, widget, funcid): + self.assertEqual( + widget.tk.splitlist(widget.tk.call('info', 'commands', funcid)), + ()) + + def test_tag_bind(self): + c = self.create() + c.pack() + item = c.create_rectangle(20, 20, 100, 100, fill='red') + self.assertEqual(c.tag_bind(item), ()) + self.assertEqual(c.tag_bind(item, ''), '') + + events = [] + def test1(e): events.append('a') + def test2(e): events.append('b') + + funcid = c.tag_bind(item, '', test1) + self.assertEqual(c.tag_bind(item), ('',)) + script = c.tag_bind(item, '') + self.assertIn(funcid, script) + self.assertCommandExist(c, funcid) + + funcid2 = c.tag_bind(item, '', test2, add=True) + script = c.tag_bind(item, '') + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(c, funcid) + self.assertCommandExist(c, funcid2) + + c.wait_visibility() + c.focus_force() + c.update() + c.event_generate('', x=50, y=50) + c.update() + self.assertEqual(events, ['a', 'b']) + + # Binding to a tag applies to all items carrying it. + c.addtag_withtag('spam', item) + events.clear() + c.tag_bind('spam', '', test1) + c.event_generate('', x=50, y=50) + c.update() + self.assertEqual(events, ['a']) + + def test_tag_unbind(self): + c = self.create() + c.pack() + item = c.create_rectangle(20, 20, 100, 100, fill='red') + + events = [] + def test1(e): events.append('a') + def test2(e): events.append('b') + + funcid = c.tag_bind(item, '', test1) + funcid2 = c.tag_bind(item, '', test2, add=True) + c.wait_visibility() + c.focus_force() + c.update() + c.event_generate('', x=50, y=50) + c.update() + self.assertEqual(events, ['a', 'b']) + + # Removing one function leaves the other in place. + c.tag_unbind(item, '', funcid) + script = c.tag_bind(item, '') + self.assertNotIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandNotExist(c, funcid) + self.assertCommandExist(c, funcid2) + events.clear() + c.event_generate('', x=50, y=50) + c.update() + self.assertEqual(events, ['b']) + + # Without a funcid all bindings for the sequence are removed. + c.tag_unbind(item, '') + self.assertEqual(c.tag_bind(item, ''), '') + self.assertEqual(c.tag_bind(item), ()) + events.clear() + c.event_generate('', x=50, y=50) + c.update() + self.assertEqual(events, []) + + self.assertRaises(TypeError, c.tag_unbind, item) + @add_configure_tests(IntegerSizeTests, StandardOptionsTests) class ListboxTest(AbstractWidgetTest, unittest.TestCase): @@ -1180,6 +1886,123 @@ def test_get(self): self.assertRaises(TypeError, lb.get, 1, 2, 3) self.assertRaises(TclError, lb.get, 2.4) + def test_size(self): + lb = self.create() + self.assertEqual(lb.size(), 0) + lb.insert(0, *('el%d' % i for i in range(8))) + self.assertEqual(lb.size(), 8) + lb.delete(0, 2) + self.assertEqual(lb.size(), 5) + self.assertRaises(TypeError, lb.size, 0) + + def test_delete(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + lb.delete(0) + self.assertEqual(lb.get(0, 'end'), + ('el1', 'el2', 'el3', 'el4', 'el5', 'el6', 'el7')) + lb.delete(2, 4) + self.assertEqual(lb.get(0, 'end'), ('el1', 'el2', 'el6', 'el7')) + lb.delete(0, 'end') + self.assertEqual(lb.size(), 0) + self.assertRaises(TclError, lb.delete, 'noindex') + self.assertRaises(TypeError, lb.delete) + + def test_index(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + self.assertEqual(lb.index(3), 3) + self.assertEqual(lb.index('end'), 8) # the number of elements + lb.activate(4) + self.assertEqual(lb.index('active'), 4) + lb.selection_anchor(2) + self.assertEqual(lb.index('anchor'), 2) + self.assertRaisesRegex(TclError, 'bad listbox index "spam"', + lb.index, 'spam') + + def test_nearest(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + lb.pack() + lb.wait_visibility() + lb.update() + # Derive the line height from the first item, which is always + # displayed (bbox() returns None for items that are not). + x, y, w, h = lb.bbox(0) + self.assertEqual(lb.nearest(y + h // 2), 0) + self.assertEqual(lb.nearest(y + 3 * h + h // 2), 3) + self.assertRaises(TclError, lb.nearest, 'spam') + self.assertRaises(TypeError, lb.nearest) + + def test_see(self): + lb = self.create(height=5) + lb.insert(0, *('el%d' % i for i in range(20))) + lb.pack() + lb.update_idletasks() + lb.see('end') + lb.update_idletasks() + self.assertEqual(lb.yview()[1], 1.0) + lb.see(0) + lb.update_idletasks() + self.assertEqual(lb.yview()[0], 0.0) + self.assertRaises(TclError, lb.see, 'spam') + + def test_activate(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + lb.activate(3) + self.assertEqual(lb.index('active'), 3) + lb.activate('end') + self.assertEqual(lb.index('active'), 7) + self.assertRaises(TclError, lb.activate, 'spam') + self.assertRaises(TypeError, lb.activate) + + def test_selection(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + self.assertEqual(lb.curselection(), ()) + self.assertFalse(lb.selection_includes(0)) + + lb.selection_set(2, 4) + lb.selection_set(6) + self.assertEqual(lb.curselection(), (2, 3, 4, 6)) + self.assertTrue(lb.selection_includes(3)) + self.assertFalse(lb.selection_includes(5)) + + lb.selection_clear(3, 4) + self.assertEqual(lb.curselection(), (2, 6)) + + lb.selection_anchor(5) + self.assertEqual(lb.index('anchor'), 5) + + # select_* are aliases of the selection_* methods. + lb.select_clear(0, 'end') + self.assertEqual(lb.curselection(), ()) + lb.select_set(1) + self.assertTrue(lb.select_includes(1)) + lb.select_anchor(1) + self.assertEqual(lb.index('anchor'), 1) + + self.assertRaisesRegex(TclError, 'bad listbox index "spam"', + lb.selection_includes, 'spam') + + def test_selection_event(self): + # Keyboard navigation changes the selection and fires the + # <> virtual event. + lb = self.create(selectmode='browse', exportselection=False) + lb.insert(0, *('el%d' % i for i in range(5))) + lb.pack() + lb.update() + events = [] + lb.bind('<>', lambda e: events.append(lb.curselection())) + lb.focus_force() + lb.activate(0) + lb.event_generate('') + lb.event_generate('') + lb.update() + self.assertEqual(events, [(1,), (2,)]) + self.assertEqual(lb.curselection(), (2,)) + @add_configure_tests(PixelSizeTests, StandardOptionsTests) class ScaleTest(AbstractWidgetTest, unittest.TestCase): @@ -1249,6 +2072,14 @@ def test_configure_to(self): self.checkFloatParam(widget, 'to', 300, 14.9, 15.1, -10, conv=float_round) + def test_identify(self): + widget = self.create() + widget.pack() + widget.update_idletasks() + self.assertIn(widget.identify(5, 5), + ('slider', 'trough1', 'trough2', '')) + self.assertRaises(TclError, widget.identify, 'a', 'b') + @add_configure_tests(PixelSizeTests, StandardOptionsTests) class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): @@ -1302,6 +2133,34 @@ def test_set(self): self.assertRaises(TypeError, sb.set, 0.6) self.assertRaises(TypeError, sb.set, 0.6, 0.7, 0.8) + def test_fraction(self): + sb = self.create() + sb.pack(fill='y', expand=True) + sb.update_idletasks() + self.assertIsInstance(sb.fraction(0, 0), float) + f = sb.fraction(0, 1000) + self.assertIsInstance(f, float) + self.assertGreaterEqual(f, 0.0) + self.assertLessEqual(f, 1.0) + self.assertRaises(TclError, sb.fraction, 'a', 'b') + self.assertRaises(TypeError, sb.fraction, 0) + + def test_delta(self): + sb = self.create() + sb.pack(fill='y', expand=True) + sb.update_idletasks() + self.assertIsInstance(sb.delta(0, 10), float) + self.assertRaises(TclError, sb.delta, 'a', 'b') + self.assertRaises(TypeError, sb.delta, 0) + + def test_identify(self): + sb = self.create() + sb.pack(fill='y', expand=True) + sb.update_idletasks() + self.assertIn(sb.identify(5, 5), + ('arrow1', 'arrow2', 'slider', 'trough1', 'trough2', '')) + self.assertRaises(TclError, sb.identify, 'a', 'b') + @add_configure_tests(PixelSizeTests, StandardOptionsTests) class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): @@ -1379,6 +2238,75 @@ def create2(self): p.add(c) return p, b, c + def test_panes(self): + p, b, c = self.create2() + self.assertEqual([str(x) for x in p.panes()], [str(b), str(c)]) + + def test_remove(self): + p, b, c = self.create2() + p.remove(b) + self.assertEqual([str(x) for x in p.panes()], [str(c)]) + p.forget(c) # forget is an alias of remove. + self.assertEqual(p.panes(), ()) + + def test_sash(self): + p, b, c = self.create2() + p.configure(width=200, height=50) + p.pack() + p.update() + x, y = p.sash_coord(0) + self.assertIsInstance(x, int) + self.assertIsInstance(y, int) + p.sash_place(0, 120, 0) + p.update() + self.assertEqual(p.sash_coord(0)[0], 120) + p.sash_mark(0) # No exception. + self.assertRaises(TclError, p.sash_coord, 5) + + def test_proxy(self): + p, b, c = self.create2() + p.configure(width=200, height=50) + p.pack() + p.update() + p.proxy_place(100, 10) + p.update() + self.assertEqual(p.proxy_coord()[0], 100) + p.proxy_forget() + p.update() + + def test_identify(self): + p, b, c = self.create2() + p.configure(width=200, height=50) + p.pack() + p.update() + x, y = p.sash_coord(0) + # A point over the sash reports the sash. + self.assertIn('sash', p.identify(x + 1, y + 5)) + # A point over a pane reports nothing. + self.assertFalse(p.identify(2, 2)) + self.assertRaises(TclError, p.identify, 'a', 'b') + + def test_add_options(self): + p = self.create() + b = tkinter.Button(p) + p.add(b, minsize=40, padx=3, sticky='ns') + self.assertEqual(p.panecget(b, 'minsize'), + 40 if self.wantobjects else '40') + self.assertEqual(p.panecget(b, 'padx'), + 3 if self.wantobjects else '3') + self.assertEqual(p.panecget(b, 'sticky'), 'ns') + self.assertRaisesRegex(TclError, 'unknown option "-spam"', + p.add, tkinter.Button(p), spam='x') + self.assertRaisesRegex(TclError, 'bad window path name "spam"', + p.add, 'spam') + + def test_paneconfigure_errors(self): + p, b, c = self.create2() + self.assertRaisesRegex(TclError, 'unknown option "-spam"', + p.paneconfigure, b, spam='x') + self.assertRaises(TclError, p.panecget, b, 'spam') + self.assertRaises(TclError, p.paneconfigure, 'spam') + def test_paneconfigure(self): p, b, c = self.create2() self.assertRaises(TypeError, p.paneconfigure) @@ -1542,6 +2470,221 @@ def test_entryconfigure_variable(self): m1.entryconfigure(1, variable=v2) self.assertEqual(str(m1.entrycget(1, 'variable')), str(v2)) + def test_add(self): + m = self.create(tearoff=False) + m.add_command(label='Command') + m.add_checkbutton(label='Checkbutton') + m.add_radiobutton(label='Radiobutton') + m.add_separator() + m.add_cascade(label='Cascade', menu=tkinter.Menu(m, tearoff=False)) + self.assertEqual(m.index('end'), 4) + self.assertEqual([m.type(i) for i in range(5)], + ['command', 'checkbutton', 'radiobutton', + 'separator', 'cascade']) + self.assertEqual(m.entrycget(0, 'label'), 'Command') + self.assertRaisesRegex(TclError, 'bad menu entry type "spam"', + m.add, 'spam') + + def test_insert(self): + m = self.create(tearoff=False) + m.add_command(label='A') + m.add_command(label='C') + m.insert_command(1, label='B') + m.insert_separator(0) + m.insert_checkbutton('end', label='D') + m.insert_radiobutton(0, label='top') + m.insert_cascade(2, label='sub', + menu=tkinter.Menu(m, tearoff=False)) + self.assertEqual( + [m.type(i) for i in range(m.index('end') + 1)], + ['radiobutton', 'separator', 'cascade', 'command', + 'command', 'command', 'checkbutton']) + self.assertEqual( + [m.entrycget(i, 'label') for i in (3, 4, 5)], + ['A', 'B', 'C']) + self.assertRaisesRegex(TclError, 'bad menu entry type "spam"', + m.insert, 0, 'spam') + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.insert_command, 'spam', label='z') + + def test_delete(self): + m = self.create(tearoff=False) + commands = [] + for label in 'ABCDE': + m.add_command(label=label, + command=lambda label=label: commands.append(label)) + # The Tcl command for a deleted item is cleaned up. + funcid = str(m.entrycget(2, 'command')) + self.assertEqual( + m.tk.splitlist(m.tk.call('info', 'commands', funcid)), (funcid,)) + + m.delete(2) # Delete a single item ('C'). + self.assertEqual([m.entrycget(i, 'label') for i in range(4)], + ['A', 'B', 'D', 'E']) + self.assertEqual( + m.tk.splitlist(m.tk.call('info', 'commands', funcid)), ()) + + m.delete(1, 2) # Delete a range ('B' and 'D'). + self.assertEqual([m.entrycget(i, 'label') for i in range(2)], + ['A', 'E']) + self.assertRaises(TypeError, m.delete) + + def test_index(self): + m = self.create(tearoff=False) + self.assertIsNone(m.index('end')) + m.add_command(label='First') + m.add_command(label='Second') + self.assertEqual(m.index('end'), 1) + self.assertEqual(m.index('last'), 1) + self.assertEqual(m.index('Second'), 1) + self.assertEqual(m.index(0), 0) + # 'active' and 'none' map to None when no item is active. + self.assertIsNone(m.index('active')) + self.assertIsNone(m.index('none')) + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.index, 'spam') + + def test_invoke(self): + m = self.create(tearoff=False) + commands = [] + m.add_command(label='Command', + command=lambda: commands.append('invoked')) + var = tkinter.IntVar(self.root) + m.add_checkbutton(label='Check', variable=var, + onvalue=1, offvalue=0) + rvar = tkinter.StringVar(self.root) + m.add_radiobutton(label='Radio', variable=rvar, value='on') + + m.invoke(0) + self.assertEqual(commands, ['invoked']) + m.invoke(1) + self.assertEqual(var.get(), 1) + m.invoke(1) + self.assertEqual(var.get(), 0) + m.invoke(2) + self.assertEqual(rvar.get(), 'on') + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.invoke, 'spam') + + def test_xposition_yposition(self): + m = self.create(tearoff=False) + m.add_command(label='First') + m.add_command(label='Second') + m.update_idletasks() + self.assertIsInstance(m.xposition(0), int) + y0 = m.yposition(0) + y1 = m.yposition(1) + self.assertIsInstance(y0, int) + self.assertLess(y0, y1) + # An out-of-range index gives the position past the last item. + self.assertEqual(m.xposition('end'), m.xposition(1)) + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.xposition, 'spam') + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.yposition, 'spam') + + def test_post_unpost(self): + m = self.create(tearoff=False) + if m._windowingsystem != 'x11': + # Posting a menu is modal on Windows and uses a native, unmapped + # menu on Aqua, so it cannot be tested synchronously there. + self.skipTest('menu posting is not testable on this platform') + m.add_command(label='First') + m.add_command(label='Second') + self.assertFalse(m.winfo_ismapped()) + + m.post(0, 0) + m.update() + self.assertTrue(m.winfo_ismapped()) + m.unpost() + m.update() + self.assertFalse(m.winfo_ismapped()) + + m.tk_popup(0, 0) + m.update() + self.assertTrue(m.winfo_ismapped()) + m.unpost() + m.update() + self.assertFalse(m.winfo_ismapped()) + + def test_postcascade(self): + m = self.create(tearoff=False) + submenu = tkinter.Menu(m, tearoff=False) + submenu.add_command(label='Item') + m.add_cascade(label='Cascade', menu=submenu) + m.add_command(label='Plain') + # No effect (but no error) when the menu is not posted, when the index + # is not a cascade entry, or when given a label. + m.postcascade(0) + m.postcascade(1) + m.postcascade('Cascade') + + with self.subTest('posted menu'): + if m._windowingsystem != 'x11': + # Posting a menu is modal on Windows and uses a native, + # unmapped menu on Aqua, so it cannot be tested synchronously + # there. + self.skipTest('menu posting is not testable on this platform') + m.post(0, 0) + m.update() + m.postcascade('Cascade') + m.update() + self.assertTrue(submenu.winfo_ismapped()) + # A non-cascade index unposts the currently posted submenu. + m.postcascade(1) + m.update() + self.assertFalse(submenu.winfo_ismapped()) + m.unpost() + + def check_entry_option(self, m, index, option, value, expected=None): + if expected is None: + expected = value + m.entryconfigure(index, **{option: value}) + self.assertEqual(str(m.entrycget(index, option)), str(expected)) + self.assertEqual(str(m.entryconfigure(index, option)[4]), str(expected)) + + def test_entry_options(self): + m = self.create(tearoff=False) + m.add_command(label='Command') + self.check_entry_option(m, 0, 'accelerator', 'Ctrl+O') + self.check_entry_option(m, 0, 'underline', 2) + self.check_entry_option(m, 0, 'state', 'disabled') + self.check_entry_option(m, 0, 'background', 'red') + self.check_entry_option(m, 0, 'foreground', 'blue') + self.check_entry_option(m, 0, 'columnbreak', 1) + self.check_entry_option(m, 0, 'hidemargin', 1) + + m.add_checkbutton(label='Checkbutton') + self.check_entry_option(m, 1, 'onvalue', 'Y') + self.check_entry_option(m, 1, 'offvalue', 'N') + self.check_entry_option(m, 1, 'indicatoron', 0) + + m.add_radiobutton(label='Radiobutton') + self.check_entry_option(m, 2, 'value', 'V') + self.check_entry_option(m, 2, 'selectcolor', 'green') + + def test_entry_options_invalid(self): + m = self.create(tearoff=False) + m.add_command(label='Command') + self.assertRaisesRegex(TclError, 'unknown option "-spam"', + m.entrycget, 0, 'spam') + self.assertRaisesRegex(TclError, 'unknown option "-spam"', + m.entryconfigure, 0, spam='x') + self.assertRaisesRegex(TclError, 'bad state "spam"', + m.entryconfigure, 0, state='spam') + # Tk < 9 reports "expected integer but got ...", while Tk 9, where + # underline accepts an index, reports "bad index ...". + self.assertRaisesRegex(TclError, + r'(expected integer but got|bad index) "spam"', + m.entryconfigure, 0, underline='spam') + self.assertRaisesRegex(TclError, 'unknown color name "spam"', + m.entryconfigure, 0, background='spam') + self.assertRaisesRegex(TclError, 'expected boolean value but got "spam"', + m.entryconfigure, 0, columnbreak='spam') + # onvalue applies only to checkbutton and radiobutton entries. + self.assertRaisesRegex(TclError, 'unknown option "-onvalue"', + m.entrycget, 0, 'onvalue') + @add_configure_tests(PixelSizeTests, StandardOptionsTests) class MessageTest(AbstractWidgetTest, unittest.TestCase): diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index ab53a20cff55392..9a878164489681a 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -3421,7 +3421,7 @@ def determine_valid_prefixes(): # some uppercase-only prefix is added. for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase): try: - eval(f'{letter}""') + eval(f'{letter}"""\n"""') # d-string needs multiline single_char_valid_prefixes.add(letter.lower()) except SyntaxError: pass @@ -3445,7 +3445,7 @@ def determine_valid_prefixes(): # because it's a valid expression: not "" continue try: - eval(f'{p}""') + eval(f'{p}"""\n"""') # d-string needs multiline # No syntax error, so p is a valid string # prefix. diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index d1831d68f0281d9..7583b61480e3709 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -427,6 +427,11 @@ def test_help_text(self): self.assertEqual(res.out, b'') self.assertIn(b'pygettext -- Python equivalent of xgettext(1)', res.err) + def test_version_text(self): + """Test that the version text is displayed.""" + res = assert_python_ok(self.script, '--version') + self.assertIn(b'pygettext.py (xgettext for Python) 1.5', res.out) + def test_error_messages(self): """Test that pygettext outputs error messages to stderr.""" stderr = self.get_stderr(dedent('''\ diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 909808825f055e2..e38d0942e463e9c 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -23,7 +23,7 @@ requires_subprocess, os_helper) from test.support.os_helper import TESTFN, temp_dir, unlink from test.support.script_helper import assert_python_ok, assert_python_failure, make_script -from test.support.import_helper import forget +from test.support.import_helper import ensure_lazy_imports, forget from test.support import force_not_colorized, force_not_colorized_test_class import json @@ -206,7 +206,7 @@ def test_recursion_error_during_traceback(self): sys.setrecursionlimit(15) def f(): - ref(lambda: 0, []) + ref(lambda: 0, ord) f() try: @@ -1815,6 +1815,18 @@ class TestKeywordTypoSuggestions(unittest.TestCase): ("[x for x\nin range(3)\nof x]", "if"), ("[123 fur x\nin range(3)\nif x]", "for"), ("for x im n:\n pass", "in"), + ("mach x:", "match"), + ("math x:", "match"), + ("match 1:\n cse 1:", "case"), + ("typ x = int", "type"), + ("typed x = int", "type"), + ("lazi import x", "lazy"), + ("lezi import x", "lazy"), + ("switch x:\n case:", "match"), + ("delete x", "del"), + ("function f():", "def"), + ("func f():", "def"), + ("void f():", "def"), ] def test_keyword_suggestions_from_file(self): @@ -4565,6 +4577,98 @@ def __init__(self): actual = self.get_suggestion(Outer(), 'target') self.assertIn("'.normal.target'", actual) + @force_not_colorized + def test_cross_language(self): + cases = [ + # (type, attr, hint_attr) + (list, 'push', 'append'), + (list, 'concat', 'extend'), + (list, 'addAll', 'extend'), + (str, 'toUpperCase', 'upper'), + (str, 'toLowerCase', 'lower'), + (str, 'trimStart', 'lstrip'), + (str, 'trimEnd', 'rstrip'), + (dict, 'keySet', 'keys'), + (dict, 'entrySet', 'items'), + (dict, 'entries', 'items'), + (dict, 'putAll', 'update'), + ] + for test_type, attr, hint_attr in cases: + with self.subTest(type=test_type.__name__, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, f"Did you mean '.{hint_attr}'?") + + cases = [ + # (type, attr, hint) + (list, 'contains', "Use 'x in list'."), + (list, 'add', "Did you mean to use a 'set' object?"), + (dict, 'put', "Use d[k] = v."), + ] + for test_type, attr, expected in cases: + with self.subTest(type=test_type, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, expected) + + @force_not_colorized + def test_cross_language_levenshtein_fallback(self): + # When no cross-language entry exists, Levenshtein still works + # (e.g., trim->strip is not in the table but Levenshtein catches it) + actual = self.get_suggestion('', 'trim') + self.assertIn("strip", actual) + + @force_not_colorized + def test_cross_language_no_hint_for_unknown_attr(self): + actual = self.get_suggestion([], 'completely_unknown_method') + self.assertNotIn("Did you mean", actual) + + @force_not_colorized + def test_cross_language_works_for_subclasses(self): + # isinstance() check means subclasses also get hints + class MyList(list): + pass + actual = self.get_suggestion(MyList(), 'push') + self.assertEndsWith(actual, "Did you mean '.append'?") + + class MyDict(dict): + pass + actual = self.get_suggestion(MyDict(), 'keySet') + self.assertEndsWith(actual, "Did you mean '.keys'?") + + @force_not_colorized + def test_cross_language_mutable_on_immutable(self): + # Mutable method on immutable type suggests the mutable counterpart + cases = [ + (tuple, 'append', "Did you mean to use a 'list' object?"), + (tuple, 'extend', "Did you mean to use a 'list' object?"), + (tuple, 'insert', "Did you mean to use a 'list' object?"), + (tuple, 'remove', "Did you mean to use a 'list' object?"), + (frozenset, 'add', "Did you mean to use a 'set' object?"), + (frozenset, 'discard', "Did you mean to use a 'set' object?"), + (frozenset, 'remove', "Did you mean to use a 'set' object?"), + (frozenset, 'update', "Did you mean to use a 'set' object?"), + (frozendict, 'update', "Did you mean to use a 'dict' object?"), + (tuple, 'clear', "Did you mean to use a 'list' object?"), + (frozenset, 'clear', "Did you mean to use a 'set' object?"), + (frozendict, 'clear', "Did you mean to use a 'dict' object?"), + ] + for test_type, attr, expected in cases: + with self.subTest(type=test_type.__name__, attr=attr): + obj = test_type() + actual = self.get_suggestion(obj, attr) + self.assertEndsWith(actual, expected) + + @force_not_colorized + def test_cross_language_float_bitwise(self): + # Bitwise operators on float suggest using int + cases = ['__or__', '__and__', '__xor__', '__lshift__', '__rshift__'] + for attr in cases: + with self.subTest(attr=attr): + actual = self.get_suggestion(1.0, attr) + self.assertIn("'int'", actual) + self.assertIn("Bitwise operators", actual) + def make_module(self, code): tmpdir = Path(tempfile.mkdtemp()) self.addCleanup(shutil.rmtree, tmpdir) @@ -5543,5 +5647,11 @@ def test_suggestion_still_works_for_non_lazy_attributes(self): self.assertNotIn(b"BAR_MODULE_LOADED", stdout) +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("traceback", {"_colorize"}) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ttk/test_extensions.py b/Lib/test/test_ttk/test_extensions.py index 669a3e184eb771d..0460f1b9aa31172 100644 --- a/Lib/test/test_ttk/test_extensions.py +++ b/Lib/test/test_ttk/test_extensions.py @@ -4,10 +4,14 @@ from tkinter import ttk from test.support import requires, gc_collect from test.test_tkinter.support import setUpModule # noqa: F401 -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest +from test.test_tkinter.support import (AbstractTkTest, AbstractDefaultRootTest, + get_tk_patchlevel, widget_eq) requires('gui') +from test.test_ttk import test_widgets + + class LabeledScaleTest(AbstractTkTest, unittest.TestCase): def tearDown(self): @@ -193,7 +197,7 @@ def test_resize(self): x.destroy() -class OptionMenuTest(AbstractTkTest, unittest.TestCase): +class OptionMenuTest(test_widgets.MenubuttonTest, unittest.TestCase): def setUp(self): super().setUp() @@ -203,6 +207,47 @@ def tearDown(self): del self.textvar super().tearDown() + def create(self, default='b', values=('a', 'b', 'c'), **kwargs): + return ttk.OptionMenu(self.root, self.textvar, default, *values, **kwargs) + + def test_bad_kwarg(self): + with self.assertRaisesRegex(tkinter.TclError, r"^unknown option -image$"): + ttk.OptionMenu(self.root, self.textvar, 'b', image='') + + def test_configure_class(self): + # Unlike a plain Menubutton, OptionMenu does not accept a class at + # construction, so only the read-only nature of the option is tested. + widget = self.create() + self.assertEqual(widget['class'], '') + errmsg = 'attempt to change read-only option' + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): + errmsg = 'Attempt to change read-only option' + self.checkInvalidParam(widget, 'class', 'Foo', errmsg=errmsg) + + def test_configure_style(self): + # Like Menubutton, but OptionMenu does not accept a class at + # construction, so the custom-class part of the standard test is omitted. + widget = self.create() + self.assertEqual(widget['style'], '') + self.checkInvalidParam(widget, 'style', 'Foo', + errmsg='Layout Foo not found') + style = ttk.Style(self.root) + style.configure('Custom.TMenubutton') + self.checkParam(widget, 'style', 'Custom.TMenubutton') + + def test_configure_menu(self): + # OptionMenu manages its own menu; ['menu'] returns that Menu widget. + widget = self.create() + self.assertIsInstance(widget['menu'], tkinter.Menu) + self.checkParam(widget, 'menu', widget['menu'], eq=widget_eq) + + def test_configure_text(self): + # The displayed text is governed by the textvariable. + widget = self.create() + self.textvar.set('a') + self.assertEqual(widget['text'], 'a') + self.textvar.set('c') + self.assertEqual(widget['text'], 'c') def test_widget_destroy(self): var = tkinter.StringVar(self.root) @@ -281,6 +326,24 @@ def cb_test(item): optmenu.destroy() + def test_set_menu(self): + optmenu = ttk.OptionMenu(self.root, self.textvar, 'a', 'a', 'b', 'c') + menu = optmenu['menu'] + self.assertEqual(menu.index('end'), 2) + + # set_menu rebuilds the menu with new values and an optional default. + optmenu.set_menu('y', 'x', 'y', 'z') + self.assertEqual(self.textvar.get(), 'y') + self.assertEqual([menu.entrycget(i, 'label') for i in range(3)], + ['x', 'y', 'z']) + + # Without a default the variable is left unchanged. + optmenu.set_menu(None, 'p', 'q') + self.assertEqual(self.textvar.get(), 'y') + self.assertEqual([menu.entrycget(i, 'label') for i in range(2)], + ['p', 'q']) + optmenu.destroy() + def test_unique_radiobuttons(self): # check that radiobuttons are unique across instances (bpo25684) items = ('a', 'b', 'c') diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index fdbaae1b644e4d2..6fbcfbdc9b9e4f5 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -6,7 +6,8 @@ from test import support from test.support import requires from test.test_tkinter.support import setUpModule # noqa: F401 -from test.test_tkinter.support import AbstractTkTest, get_tk_patchlevel +from test.test_tkinter.support import (AbstractTkTest, get_tk_patchlevel, + requires_tk) requires('gui') @@ -124,6 +125,41 @@ def test_theme_use(self): self.style.theme_use(curr_theme) + @requires_tk(9, 0) + def test_theme_styles(self): + # The 'default' theme is always available and defines the base styles. + default_styles = self.style.theme_styles('default') + self.assertIsInstance(default_styles, tuple) + self.assertIn('.', default_styles) + self.assertIn('TButton', default_styles) + + # Without an argument the current theme is used. + styles = self.style.theme_styles() + self.assertIsInstance(styles, tuple) + self.assertIn('.', styles) + + for theme in self.style.theme_names(): + self.assertIsInstance(self.style.theme_styles(theme), tuple) + + self.assertRaises(tkinter.TclError, + self.style.theme_styles, 'nonexistingname') + + def test_theme_settings(self): + style = self.style + theme = style.theme_use() + style.theme_settings(theme, { + 'Test.TLabel': { + 'configure': {'foreground': 'red', 'background': 'blue'}, + 'map': {'foreground': [('active', 'green')]}, + }, + }) + self.assertEqual(style.lookup('Test.TLabel', 'foreground'), 'red') + self.assertEqual(style.lookup('Test.TLabel', 'background'), 'blue') + self.assertEqual(style.map('Test.TLabel', 'foreground'), + [('active', 'green')]) + self.assertRaises(tkinter.TclError, style.theme_settings, + 'nonexistingname', {}) + def test_configure_custom_copy(self): style = self.style diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index 8cce9aed9d514f4..bffcfcff526c668 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -789,6 +789,7 @@ def cb_test(): self.assertEqual(str(cbtn['variable']), str(cbtn2['variable'])) +@add_configure_tests(StandardTtkOptionsTests) class MenubuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'compound', 'cursor', 'direction', @@ -973,6 +974,27 @@ def test_configure_value(self): test_configure_wraplength = requires_tk(8, 7)(StandardOptionsTests.test_configure_wraplength) + def test_step(self): + widget = self.create(maximum=100, mode='determinate') + self.assertEqual(float(widget['value']), 0.0) + widget.step() # The default increment is 1.0. + self.assertEqual(float(widget['value']), 1.0) + widget.step(5) + self.assertEqual(float(widget['value']), 6.0) + widget.step(-2) + self.assertEqual(float(widget['value']), 4.0) + + def test_start_stop(self): + widget = self.create(maximum=100, mode='determinate') + widget.pack() + widget.start() # Schedule autoincrement; no exception. + widget.update() + widget.stop() # Cancel it. + # After stopping, the value no longer changes. + value = float(widget['value']) + widget.update() + self.assertEqual(float(widget['value']), value) + @unittest.skipIf(sys.platform == 'darwin', 'ttk.Scrollbar is special on MacOSX') @@ -1286,6 +1308,19 @@ def test_configure_command(self): self.spin.update() self.assertEqual(len(success), 2) + def test_increment_decrement_events(self): + # Clicking the arrows fires the <> and <> + # virtual events. + events = [] + self.spin.bind('<>', lambda e: events.append('increment')) + self.spin.bind('<>', lambda e: events.append('decrement')) + self.spin.update() + self._click_increment_arrow() + self.spin.update() + self._click_decrement_arrow() + self.spin.update() + self.assertEqual(events, ['increment', 'decrement']) + def test_configure_to(self): self.spin['from'] = 0 self.spin['to'] = 5 @@ -1639,6 +1674,50 @@ def test_exists(self): # in the tcl interpreter since tk requires an item. self.assertRaises(tkinter.TclError, self.tv.exists, None) + def test_parent(self): + a = self.tv.insert('', 'end') + b = self.tv.insert(a, 'end') + self.assertEqual(self.tv.parent(b), a) + self.assertEqual(self.tv.parent(a), '') + self.assertRaises(tkinter.TclError, self.tv.parent, 'nonexistent') + + def test_next_prev(self): + a = self.tv.insert('', 'end') + b = self.tv.insert('', 'end') + c = self.tv.insert('', 'end') + self.assertEqual(self.tv.next(a), b) + self.assertEqual(self.tv.next(b), c) + self.assertEqual(self.tv.next(c), '') + self.assertEqual(self.tv.prev(c), b) + self.assertEqual(self.tv.prev(b), a) + self.assertEqual(self.tv.prev(a), '') + self.assertRaises(tkinter.TclError, self.tv.next, 'nonexistent') + self.assertRaises(tkinter.TclError, self.tv.prev, 'nonexistent') + + def test_see(self): + a = self.tv.insert('', 'end') + b = self.tv.insert(a, 'end') + # see() opens all of the item's ancestors. + self.assertFalse(self.tv.tk.getboolean(self.tv.item(a, 'open'))) + self.tv.see(b) + self.assertTrue(self.tv.tk.getboolean(self.tv.item(a, 'open'))) + self.assertRaises(tkinter.TclError, self.tv.see, 'nonexistent') + + def test_identify_element(self): + self.tv.pack() + self.tv.wait_visibility() + parent = self.tv.insert('', 'end', text='parent') + self.tv.insert(parent, 'end', text='child') + self.tv.update() + x, y, w, h = self.tv.bbox(parent) + # The Treeitem.indicator element is packed at the left of the row in + # the Item layout on every platform and theme. + element = self.tv.identify_element(x + 8, y + h // 2) + self.assertRegex(element, r'.*indicator\z') + # The empty string is returned outside the widget. + self.assertEqual(self.tv.identify_element(-1, -1), '') + self.assertRaises(tkinter.TclError, self.tv.identify_element, None, 5) + def test_focus(self): # nothing is focused right now self.assertEqual(self.tv.focus(), '') @@ -1880,6 +1959,37 @@ def test_selection(self): self.tv.selection_toggle((c1, c3)) self.assertEqual(self.tv.selection(), (c3, item2)) + def test_virtual_events(self): + # Keyboard navigation fires the <>, <> + # and <> virtual events. + parent = self.tv.insert('', 'end') + self.tv.insert(parent, 'end') + item2 = self.tv.insert('', 'end') + self.tv.pack() + self.tv.update() + selects, opens, closes = [], [], [] + self.tv.bind('<>', + lambda e: selects.append(self.tv.selection())) + self.tv.bind('<>', lambda e: opens.append(self.tv.focus())) + self.tv.bind('<>', lambda e: closes.append(self.tv.focus())) + self.tv.focus_force() + self.tv.focus(parent) + self.tv.selection_set(parent) + self.tv.update() + + self.tv.event_generate('') # Open the focused parent. + self.tv.update() + self.assertEqual(opens, [parent]) + + self.tv.event_generate('') # Close it again. + self.tv.update() + self.assertEqual(closes, [parent]) + + self.tv.event_generate('') # Move the selection. + self.tv.update() + self.assertEqual(self.tv.selection(), (item2,)) + self.assertIn((item2,), selects) + def test_set(self): self.tv['columns'] = ['A', 'B'] item = self.tv.insert('', 'end', values=['a', 'b']) @@ -1971,6 +2081,264 @@ def test_tag_has(self): self.assertEqual(self.tv.tag_has('tag2'), (item2,)) self.assertEqual(self.tv.tag_has('tag3'), ()) + def build_tree(self): + # a -> a1, a2 -> a2x; b + tv = self.tv + tv.insert('', 'end', 'a') + tv.insert('a', 'end', 'a1') + tv.insert('a', 'end', 'a2') + tv.insert('a2', 'end', 'a2x') + tv.insert('', 'end', 'b') + return tv + + @requires_tk(9, 1) + def test_depth(self): + tv = self.build_tree() + self.assertEqual(tv.depth('a'), 1) + self.assertEqual(tv.depth('a2'), 2) + self.assertEqual(tv.depth('a2x'), 3) + + @requires_tk(9, 1) + def test_haschildren(self): + tv = self.build_tree() + self.assertTrue(tv.haschildren('a')) + self.assertTrue(tv.haschildren('a2')) + self.assertFalse(tv.haschildren('a1')) + self.assertFalse(tv.haschildren('b')) + + @requires_tk(9, 1) + def test_identifier(self): + tv = self.build_tree() + self.assertEqual(tv.identifier('a', 0), 'a1') + self.assertEqual(tv.identifier('a', 1), 'a2') + self.assertEqual(tv.identifier('', 0), 'a') + self.assertEqual(tv.identifier('', 1), 'b') + + @requires_tk(9, 1) + def test_size(self): + tv = self.build_tree() + self.assertEqual(tv.size(''), 2) + self.assertEqual(tv.size('a'), 2) + self.assertEqual(tv.size('a', recurse=True), 3) + self.assertEqual(tv.size('b'), 0) + tv.hide('a1') + self.assertEqual(tv.size('a'), 1) + self.assertEqual(tv.size('a', hidden=True), 2) + + @requires_tk(9, 1) + def test_range(self): + tv = self.build_tree() + tv.expand('a', recurse=True) + self.assertEqual(tv.range('a', 'b'), ('a', 'a1', 'a2', 'a2x', 'b')) + self.assertEqual(tv.range('a1', 'a2'), ('a1', 'a2')) + self.assertEqual(tv.range('a', 'a'), ('a',)) + tv.hide('a1') + self.assertEqual(tv.range('a', 'b'), ('a', 'a2', 'a2x', 'b')) + self.assertEqual(tv.range('a', 'b', hidden=True), + ('a', 'a1', 'a2', 'a2x', 'b')) + + @requires_tk(9, 1) + def test_after_before_item(self): + tv = self.build_tree() + tv.expand('a', recurse=True) + self.assertEqual(tv.after_item('a'), 'a1') + self.assertEqual(tv.after_item('a1'), 'a2') + self.assertEqual(tv.after_item('a2x'), 'b') + self.assertEqual(tv.after_item('b'), '') + self.assertEqual(tv.before_item('b'), 'a2x') + self.assertEqual(tv.before_item('a1'), 'a') + self.assertEqual(tv.before_item('a'), '') + # With recurse=False the search stays at the sibling level. + self.assertEqual(tv.after_item('a', recurse=False), 'b') + self.assertEqual(tv.before_item('b', recurse=False), 'a') + # next/prev == after_item/before_item with hidden=True, recurse=False. + self.assertEqual(tv.after_item('a', hidden=True, recurse=False), + tv.next('a')) + self.assertEqual(tv.before_item('b', hidden=True, recurse=False), + tv.prev('b')) + + def test_expand_collapse(self): + # Without recurse this works on all Tk versions (emulated before 9.1). + tv = self.build_tree() + tv.expand('a') + self.assertTrue(tv.item('a', 'open')) + self.assertFalse(tv.item('a2', 'open')) + tv.collapse('a') + self.assertFalse(tv.item('a', 'open')) + # Several items at once, as separate arguments or as a list. + tv.expand('a', 'a2') + self.assertTrue(tv.item('a', 'open')) + self.assertTrue(tv.item('a2', 'open')) + tv.collapse(['a', 'a2']) + self.assertFalse(tv.item('a', 'open')) + self.assertFalse(tv.item('a2', 'open')) + + @requires_tk(9, 1) + def test_expand_collapse_recurse(self): + tv = self.build_tree() + tv.expand('a', recurse=True) + self.assertTrue(tv.item('a', 'open')) + self.assertTrue(tv.item('a2', 'open')) + tv.collapse('a', recurse=True) + self.assertFalse(tv.item('a', 'open')) + self.assertFalse(tv.item('a2', 'open')) + + @requires_tk(9, 1) + def test_hide_unhide(self): + tv = self.build_tree() + tv.expand('a', recurse=True) + tv.hide('a1') + self.assertEqual(tv.range('a', 'b'), ('a', 'a2', 'a2x', 'b')) + tv.unhide('a1') + self.assertEqual(tv.range('a', 'b'), ('a', 'a1', 'a2', 'a2x', 'b')) + tv.hide('a1', 'a2') + self.assertEqual(tv.range('a', 'b'), ('a', 'b')) + tv.unhide(['a1', 'a2']) + self.assertEqual(tv.range('a', 'b'), ('a', 'a1', 'a2', 'a2x', 'b')) + + @requires_tk(9, 1) + def test_visible(self): + tv = self.build_tree() + tv.pack() + self.addCleanup(tv.pack_forget) + self.root.update_idletasks() + self.assertTrue(tv.visible('a')) + self.assertFalse(tv.visible('a2x')) # ancestors are closed + tv.expand('a', recurse=True) + self.root.update_idletasks() + self.assertTrue(tv.visible('a2x')) + tv.hide('a2') + self.root.update_idletasks() + self.assertFalse(tv.visible('a2x')) # an ancestor is hidden + + @requires_tk(9, 1) + def test_current(self): + tv = self.build_tree() + # No item is under the mouse pointer during the test. + self.assertEqual(tv.current(), ()) + + @requires_tk(9, 0) + def test_detached(self): + tv = self.build_tree() + self.assertEqual(tv.detached(), ()) + self.assertFalse(tv.detached('a')) + tv.detach('a') + self.assertEqual(tv.detached(), ('a',)) # not the descendants + self.assertTrue(tv.detached('a')) + self.assertFalse(tv.detached('b')) + tv.move('a', '', 'end') # reattach + self.assertEqual(tv.detached(), ()) + + @requires_tk(9, 1) + def test_detached_all(self): + # The -all form and the ancestor-aware item query require Tk 9.1. + tv = self.build_tree() + self.assertEqual(tv.detached_all(), ()) + tv.detach('a') + self.assertEqual(set(tv.detached_all()), + {'a', 'a1', 'a2', 'a2x'}) # with descendants + self.assertEqual(tv.detached(), ('a',)) # without descendants + self.assertTrue(tv.detached('a2x')) # an ancestor is detached + + @requires_tk(9, 1) + def test_cellfocus(self): + tv = self.create(columns=('x',)) + tv.insert('', 'end', 'a', values=('1',)) + self.assertEqual(tv.cellfocus(), ()) + tv.cellfocus(('a', 'x')) + self.assertEqual(tv.cellfocus(), ('a', 'x')) + tv.cellfocus('') + self.assertEqual(tv.cellfocus(), ()) + + @requires_tk(9, 1) + def test_sort(self): + tv = self.create(columns=('x',)) + for name, value in [('c', '3'), ('a', '1'), ('b', '2')]: + tv.insert('', 'end', name, values=(value,)) + tv.sort('', column='x', integer=True) + self.assertEqual(tv.get_children(), ('a', 'b', 'c')) + tv.sort('', column='x', integer=True, decreasing=True) + self.assertEqual(tv.get_children(), ('c', 'b', 'a')) + tv.sort('', column='x', command=lambda p, q: (p > q) - (p < q)) + self.assertEqual(tv.get_children(), ('a', 'b', 'c')) + + @requires_tk(9, 1) + def test_search(self): + tv = self.create(columns=('x',)) + for name, value in [('a', '1'), ('b', '2'), ('c', '2')]: + tv.insert('', 'end', name, values=(value,)) + self.assertEqual(tv.search('', '2', columns=('x',)), 'b') + self.assertEqual(tv.search('', 'z', columns=('x',)), '') + self.assertEqual(tv.search('', '2', columns=('x',), backwards=True), 'c') + self.assertEqual(tv.search_all('', '2', columns=('x',)), ('b', 'c')) + self.assertEqual(tv.search_all('', '?', columns=('x',), glob=True), + ('a', 'b', 'c')) + + @requires_tk(9, 1) + def test_search_cell(self): + tv = self.create(columns=('x',)) + for name, value in [('a', '1'), ('b', '2'), ('c', '2')]: + tv.insert('', 'end', name, values=(value,)) + self.assertEqual(tv.search_cell('', '2', columns=('x',)), ('b', 'x')) + self.assertEqual(tv.search_cell('', 'z', columns=('x',)), ()) + self.assertEqual(tv.search_all_cells('', '2', columns=('x',)), + (('b', 'x'), ('c', 'x'))) + + @requires_tk(9, 1) + def test_cellselection(self): + tv = self.create(columns=('x', 'y')) + for name in 'ab': + tv.insert('', 'end', name, values=('1', '2')) + self.assertEqual(tv.cellselection(), ()) + self.assertFalse(tv.cellselection_present()) + tv.cellselection_set(('a', 'x'), ('b', 'y')) + self.assertEqual(set(tv.cellselection()), {('a', 'x'), ('b', 'y')}) + self.assertTrue(tv.cellselection_present()) + self.assertTrue(tv.cellselection_includes(('a', 'x'))) + self.assertFalse(tv.cellselection_includes(('a', 'y'))) + tv.cellselection_add(('a', 'y')) + tv.cellselection_remove(('b', 'y')) + self.assertEqual(set(tv.cellselection()), {('a', 'x'), ('a', 'y')}) + # A single list of cells is also accepted. + tv.cellselection_set([('b', 'x'), ('b', 'y')]) + self.assertEqual(set(tv.cellselection()), {('b', 'x'), ('b', 'y')}) + tv.cellselection_set() # clear + self.assertEqual(tv.cellselection(), ()) + self.assertEqual(tv.cellselection_anchor(), ()) + tv.cellselection_anchor(('a', 'x')) + self.assertEqual(tv.cellselection_anchor(), ('a', 'x')) + tv.cellselection_anchor('') + self.assertEqual(tv.cellselection_anchor(), ()) + + @requires_tk(9, 1) + def test_cellselection_range(self): + tv = self.create(columns=('x', 'y', 'z')) + for name in 'abc': + tv.insert('', 'end', name, values=('1', '2', '3')) + tv.cellselection_set_range(('a', 'x'), ('b', 'y')) + self.assertEqual(set(tv.cellselection()), + {('a', 'x'), ('a', 'y'), ('b', 'x'), ('b', 'y')}) + tv.cellselection_add_range(('c', 'z'), ('c', 'z')) + self.assertIn(('c', 'z'), tv.cellselection()) + tv.cellselection_remove_range(('a', 'x'), ('b', 'x')) + self.assertEqual(set(tv.cellselection()), + {('a', 'y'), ('b', 'y'), ('c', 'z')}) + + @requires_tk(9, 1) + def test_tag_cell(self): + tv = self.create(columns=('x', 'y')) + for name in 'ab': + tv.insert('', 'end', name, values=('1', '2')) + self.assertEqual(tv.tag_cell_has('hot'), ()) + tv.tag_cell_add('hot', ('a', 'x'), ('b', 'y')) + self.assertTrue(tv.tag_cell_has('hot', ('a', 'x'))) + self.assertFalse(tv.tag_cell_has('hot', ('a', 'y'))) + self.assertEqual(set(tv.tag_cell_has('hot')), {('a', 'x'), ('b', 'y')}) + tv.tag_cell_remove('hot', ('a', 'x')) + self.assertEqual(tv.tag_cell_has('hot'), (('b', 'y'),)) + tv.tag_cell_remove('hot') # from all cells + self.assertEqual(tv.tag_cell_has('hot'), ()) + @add_configure_tests(StandardTtkOptionsTests) class SeparatorTest(AbstractWidgetTest, unittest.TestCase): diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index d459f497e333e64..b751f825bb97d59 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -485,6 +485,13 @@ def test_comprehension_in_annotation(self): ns = run_code("x: [y for y in range(10)]") self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + def test_class_annotation_dunder_classdict(self): + ns = run_code(""" + class C: + __classdict__: int + """) + self.assertEqual(ns["C"].__annotations__, {"__classdict__": int}) + def test_future_annotations(self): code = """ from __future__ import annotations diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 22ad9f6243eda91..849a2afd8ed7986 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,4 +1,5 @@ """ Tests for the internal type cache in CPython. """ +import collections.abc import dis import unittest import warnings @@ -114,6 +115,25 @@ class HolderSub(Holder): Holder.set_value() HolderSub.value + def test_abc_register_invalidates_subclass_versions(self): + class Parent: + pass + + class Child(Parent): + pass + + type_assign_version(Parent) + type_assign_version(Child) + parent_version = type_get_version(Parent) + child_version = type_get_version(Child) + if parent_version == 0 or child_version == 0: + self.skipTest("Could not assign valid type versions") + + collections.abc.Mapping.register(Parent) + + self.assertEqual(type_get_version(Parent), 0) + self.assertEqual(type_get_version(Child), 0) + @support.cpython_only class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 84c1b9541367363..b6588269e1d0c65 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1,4 +1,5 @@ import annotationlib +import inspect import textwrap import types import unittest @@ -1446,6 +1447,30 @@ def f[T: int = int, **P = int, *Ts = int](): pass self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') + def test_signature(self): + # gh-151665: the ".format" parameter of compiler-generated evaluators + # used to break inspect.signature(). It should show up as "format". + type Alias = int + def f[T: int = int, **P = int, *Ts = int](): pass + T, P, Ts = f.__type_params__ + def g[U: (int, str)](): pass + U, = g.__type_params__ + cases = [ + Alias.evaluate_value, + T.evaluate_bound, + T.evaluate_default, + P.evaluate_default, + Ts.evaluate_default, + U.evaluate_constraints, + ] + for case in cases: + with self.subTest(case=case): + sig = inspect.signature(case) + self.assertEqual(str(sig), '(format=1, /)') + param, = sig.parameters.values() + self.assertEqual(param.name, 'format') + self.assertIs(param.kind, inspect.Parameter.POSITIONAL_ONLY) + def test_constraints(self): def f[T: (int, str)](): pass T, = f.__type_params__ diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 5d19e3706802dd4..ed07503cd63f12b 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -1316,6 +1316,11 @@ def test_infer_variance(self): def test_bound(self): Ts_bound = TypeVarTuple('Ts_bound', bound=int) self.assertIs(Ts_bound.__bound__, int) + Ts_tuple_bound = TypeVarTuple('Ts_tuple_bound', bound=(int, str)) + self.assertEqual(Ts_tuple_bound.__bound__, (int, str)) + obj = object() + Ts_object = TypeVarTuple('Ts_object', bound=obj) + self.assertIs(Ts_object.__bound__, obj) Ts_no_bound = TypeVarTuple('Ts_no_bound') self.assertIsNone(Ts_no_bound.__bound__) @@ -5848,6 +5853,27 @@ def foo(x: T): foo(42) + def test_genericalias_instance_isclass(self): + # test against user-defined generic classes + T = TypeVar('T') + + class Node(Generic[T]): + def __init__(self, label: T, + left: 'Node[T] | None' = None, + right: 'Node[T] | None' = None): + self.label = label + self.left = left + self.right = right + + self.assertTrue(inspect.isclass(Node)) + self.assertFalse(inspect.isclass(Node[int])) + self.assertFalse(inspect.isclass(Node[str])) + + # test against standard generic classes + self.assertFalse(inspect.isclass(set[int])) + self.assertFalse(inspect.isclass(list[bytes])) + self.assertFalse(inspect.isclass(dict[str, str])) + def test_implicit_any(self): T = TypeVar('T') @@ -10513,6 +10539,17 @@ def test_paramspec_in_nested_generics(self): self.assertEqual(G2[[int, str], float], list[C]) self.assertEqual(G3[[int, str], float], list[C] | int) + def test_paramspec_bound(self): + P = ParamSpec('P', bound=[int, str]) + self.assertEqual(P.__bound__, [int, str]) + P2 = ParamSpec('P2', bound=(int, str)) + self.assertEqual(P2.__bound__, (int, str)) + obj = object() + P3 = ParamSpec('P3', bound=obj) + self.assertIs(P3.__bound__, obj) + P4 = ParamSpec('P4') + self.assertIs(P4.__bound__, None) + def test_paramspec_gets_copied(self): # bpo-46581 P = ParamSpec('P') @@ -11131,6 +11168,10 @@ def test_no_attributes(self): with self.assertRaises(AttributeError): type(NoDefault).foo + def test_no_subclassing(self): + with self.assertRaises(TypeError): + class Test(type(NoDefault)): ... + class AllTests(BaseTestCase): """Tests for __all__.""" diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 8ecb0df2f8e5ddc..ad25be3da8cb347 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -616,6 +616,34 @@ def test_issue10254(self): b = 'C\u0338' * 20 + '\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079: avoid quadratic canonical ordering. + payload = "a" + ("\u0300\u0327" * 32) + nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32) + nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300" + short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10) + short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9) + long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30) + long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + def test_issue29456(self): # Fix #29456 u1176_str_a = '\u1100\u1176\u11a8' @@ -1106,6 +1134,22 @@ def test_failed_import_during_compiling(self): "(can't load unicodedata module)" self.assertIn(error, result.err.decode("ascii")) + def test_unicodedata_unload_reload(self): + # gh-149449: dropping unicodedata and running gc must not leave the + # cached _ucnhash_CAPI pointer dangling. + code = ( + "import gc, sys\n" + "assert '\\N{GRINNING FACE}'.encode(" + " 'ascii', errors='namereplace') == b'\\\\N{GRINNING FACE}'\n" + "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER A}'\", '', 'exec')\n" + "del sys.modules['unicodedata']\n" + "gc.collect()\n" + "assert '\\N{WINKING FACE}'.encode(" + " 'ascii', errors='namereplace') == b'\\\\N{WINKING FACE}'\n" + "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER B}'\", '', 'exec')\n" + ) + script_helper.assert_python_ok("-c", code) + def test_decimal_numeric_consistent(self): # Test that decimal and numeric are consistent, # i.e. if a character has a decimal value, diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index cdff6d1a20c8dfd..f4e50a3dccb2bc2 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,3 +1,4 @@ +import abc import functools import sys import types @@ -98,6 +99,22 @@ def test_loadTestsFromTestCase__from_FunctionTestCase(self): self.assertIsInstance(suite, loader.suiteClass) self.assertEqual(list(suite), []) + # "Do not load any tests from a TestCase-derived class that is an abstract + # base class." + def test_loadTestsFromTestCase__from_abc_TestCase(self): + class FooBase(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test(self): ... + class Foo(FooBase): + def test(self): pass + + empty_suite = unittest.TestSuite() + + loader = unittest.TestLoader() + suite = loader.loadTestsFromTestCase(Foo) + self.assertEqual(loader.loadTestsFromTestCase(FooBase), empty_suite) + self.assertEqual(list(suite), [Foo('test')]) + ################################################################ ### /Tests for TestLoader.loadTestsFromTestCase @@ -252,6 +269,24 @@ def load_tests(loader, tests, pattern): self.assertRaisesRegex(TypeError, "some failure", test.m) + # Check that loadTestsFromModule skips abstract base classes derived from + # TestCase, which can't be instantiated. + def test_loadTestsFromModule__skip_abc_TestCase(self): + m = types.ModuleType('m') + class MyTestCaseBase(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test(self): + ... + class MyTestCase(MyTestCaseBase): + def test(self): + pass + m.testcase_1 = MyTestCaseBase + m.testcase_2 = MyTestCase + loader = unittest.TestLoader() + suite = loader.loadTestsFromModule(m) + expected = [loader.suiteClass([MyTestCase('test')])] + self.assertEqual(list(suite), expected) + ################################################################ ### /Tests for TestLoader.loadTestsFromModule() @@ -1052,6 +1087,22 @@ def test_loadTestsFromNames__module_not_loaded(self): if module_name in sys.modules: del sys.modules[module_name] + # "The specifier should not refer to a test method in a TestCase-derived + # subclass that is an abstract base class" + def test_loadTestsFromNames__testmethod_in_abc_TestCase(self): + m = types.ModuleType('m') + class Foo(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test_1(self): ... + def test_2(self): pass + m.Foo = Foo + + loader = unittest.TestLoader() + for name in 'Foo.test_1', 'Foo.test_2': + with self.subTest(name=name), self.assertRaisesRegex(TypeError, + "Cannot instantiate abstract test case Foo"): + loader.loadTestsFromNames([name], m) + ################################################################ ### /Tests for TestLoader.loadTestsFromNames() diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 78461abcd69f337..58ae85fb268042e 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -301,15 +301,235 @@ def test_sysconfig(self): self.assertEqual(out.strip(), expected, err) for attr, expected in ( ('executable', self.envpy()), - # Usually compare to sys.executable, but if we're running in our own - # venv then we really need to compare to our base executable - ('_base_executable', sys._base_executable), + # Usually compare to sys.prefix, but if we're running in our own + # venv then we really need to compare to our base prefix + ('base_prefix', sys.base_prefix), ): with self.subTest(attr): cmd[2] = f'import sys; print(sys.{attr})' out, err = check_output(cmd, encoding='utf-8') self.assertEqual(out.strip(), expected, err) + @requireVenvCreate + def test_version_mismatch_warning(self): + """ + Test that a warning is emitted when running a venv created for a + different minor Python version. + """ + rmtree(self.env_dir) + + wrong_minor = sys.version_info.minor + 1 + self.run_with_capture(venv.create, self.env_dir, with_pip=False) + + cfg_path = self.get_env_file('pyvenv.cfg') + with open(cfg_path, 'r', encoding='utf-8') as f: + cfg_content = f.read() + + new_version = f"{sys.version_info.major}.{wrong_minor}" + if 'version =' in cfg_content: + cfg_content = re.sub(r'version = \d+\.\d+', f'version = {new_version}', cfg_content) + + cfg_content += f'\nversion_info = {new_version}\n' + + with open(cfg_path, 'w', encoding='utf-8') as f: + f.write(cfg_content) + + envpy = self.envpy(real_env_dir=True) + + proc = subprocess.run( + [envpy, '-c', 'import sys; print("done")'], + capture_output=True, + text=True, + env={**os.environ, "PYTHONHOME": ""} + ) + + self.assertIn(f"Python {sys.version_info.major}.{wrong_minor}", proc.stderr) + self.assertIn("Consider running `python -m venv --upgrade`", proc.stderr) + + @requireVenvCreate + def test_version_info_mismatch_warning(self): + """ + Test that a warning is emitted when version_info (used by virtualenv) + indicates a different minor version. + """ + rmtree(self.env_dir) + wrong_minor = sys.version_info.minor + 1 + self.run_with_capture(venv.create, self.env_dir, with_pip=False) + + cfg_path = self.get_env_file('pyvenv.cfg') + with open(cfg_path, 'r', encoding='utf-8') as f: + cfg_content = f.read() + + # Add only version_info, don't modify version + new_version = f"{sys.version_info.major}.{wrong_minor}" + cfg_content += f'\nversion_info = {new_version}\n' + + with open(cfg_path, 'w', encoding='utf-8') as f: + f.write(cfg_content) + + envpy = self.envpy(real_env_dir=True) + proc = subprocess.run( + [envpy, '-c', 'import sys; print("done")'], + capture_output=True, + text=True, + env={**os.environ, "PYTHONHOME": ""} + ) + + self.assertIn(f"Python {sys.version_info.major}.{wrong_minor}", proc.stderr) + self.assertIn("Consider running `python -m venv --upgrade`", proc.stderr) + + @requireVenvCreate + def test_version_match_no_warning(self): + """ + Test that no warning is emitted when the venv version matches. + """ + rmtree(self.env_dir) + + self.run_with_capture(venv.create, self.env_dir, with_pip=False) + cfg_path = self.get_env_file('pyvenv.cfg') + with open(cfg_path, 'r', encoding='utf-8') as f: + cfg_content = f.read() + expected_version = f"{sys.version_info.major}.{sys.version_info.minor}" + + with open(cfg_path, 'w', encoding='utf-8') as f: + f.write(cfg_content) + envpy = self.envpy(real_env_dir=True) + proc = subprocess.run( + [envpy, '-c', 'import sys; print("done")'], + capture_output=True, + text=True, + env={**os.environ, "PYTHONHOME": ""} + ) + + self.assertNotIn("Consider running `python -m venv --upgrade`", proc.stderr) + + @requireVenvCreate + def test_malformed_version_warning(self): + """ + Test that a warning is emitted on malformed version string + in pyenv.cfg + """ + rmtree(self.env_dir) + + self.run_with_capture(venv.create, self.env_dir, with_pip=False) + + cfg_path = self.get_env_file('pyvenv.cfg') + with open(cfg_path, 'r', encoding='utf-8') as f: + cfg_content = f.read() + + malformed_version = "not.a.version" + if 'version =' in cfg_content: + cfg_content = re.sub(r'version = .+', f'version = {malformed_version}', cfg_content) + + with open(cfg_path, 'w', encoding='utf-8') as f: + f.write(cfg_content) + + envpy = self.envpy(real_env_dir=True) + proc = subprocess.run( + [envpy, '-c', 'import sys; print("done")'], + capture_output=True, + text=True, + env={**os.environ, "PYTHONHOME": ""} + ) + self.assertIn("Malformed version string", proc.stderr) + self.assertIn(malformed_version, proc.stderr) + + @requireVenvCreate + def test_malformed_version_info_warning(self): + """ + Test that a warning is emitted on malformed version_info string + in pyenv.cfg + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, with_pip=False) + + cfg_path = self.get_env_file('pyvenv.cfg') + with open(cfg_path, 'r', encoding='utf-8') as f: + cfg_content = f.read() + + malformed_version = "invalid.version" + cfg_content += f'\nversion_info = {malformed_version}\n' + + with open(cfg_path, 'w', encoding='utf-8') as f: + f.write(cfg_content) + + envpy = self.envpy(real_env_dir=True) + proc = subprocess.run( + [envpy, '-c', 'import sys; print("done")'], + capture_output=True, + text=True, + env={**os.environ, "PYTHONHOME": ""} + ) + + self.assertIn("Malformed version_info string", proc.stderr) + self.assertIn(malformed_version, proc.stderr) + + @requireVenvCreate + def test_conflicting_version_fields(self): + """ + Test behavior when both version and version_info are present + but contain different values. Should warn based on first mismatch found. + """ + rmtree(self.env_dir) + wrong_minor = sys.version_info.minor + 1 + self.run_with_capture(venv.create, self.env_dir, with_pip=False) + + cfg_path = self.get_env_file('pyvenv.cfg') + with open(cfg_path, 'r', encoding='utf-8') as f: + cfg_content = f.read() + + version_wrong = f"{sys.version_info.major}.{wrong_minor}" + if 'version =' in cfg_content: + cfg_content = re.sub(r'version = \d+\.\d+', f'version = {version_wrong}', cfg_content) + + version_info_wrong = f"{sys.version_info.major}.{wrong_minor + 1}" + cfg_content += f'\nversion_info = {version_info_wrong}\n' + + with open(cfg_path, 'w', encoding='utf-8') as f: + f.write(cfg_content) + + envpy = self.envpy(real_env_dir=True) + proc = subprocess.run( + [envpy, '-c', 'import sys; print("done")'], + capture_output=True, + text=True, + env={**os.environ, "PYTHONHOME": ""} + ) + + self.assertIn("Consider running `python -m venv --upgrade`", proc.stderr) + self.assertEqual(proc.stderr.count("Consider running `python -m venv --upgrade`"), 1) + + @requireVenvCreate + def test_different_major_version_no_warning(self): + """ + Test that no warning is emitted when major version differs. + The warning should only trigger for same major, different minor. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, with_pip=False) + + cfg_path = self.get_env_file('pyvenv.cfg') + with open(cfg_path, 'r', encoding='utf-8') as f: + cfg_content = f.read() + + different_major = sys.version_info.major + 1 + new_version = f"{different_major}.{sys.version_info.minor}" + + if 'version =' in cfg_content: + cfg_content = re.sub(r'version = \d+\.\d+', f'version = {new_version}', cfg_content) + with open(cfg_path, 'w', encoding='utf-8') as f: + f.write(cfg_content) + + envpy = self.envpy(real_env_dir=True) + proc = subprocess.run( + [envpy, '-c', 'import sys; print("done")'], + capture_output=True, + text=True, + env={**os.environ, "PYTHONHOME": ""} + ) + + self.assertNotIn("Consider running `python -m venv --upgrade`", proc.stderr) + @requireVenvCreate @unittest.skipUnless(can_symlink(), 'Needs symlinks') def test_sysconfig_symlinks(self): @@ -500,6 +720,7 @@ def test_executable_symlinks(self): # gh-124651: test quoted strings @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + @unittest.skipIf(sys.platform == 'cygwin', 'fail to locate cygpython DLL') def test_special_chars_bash(self): """ Test that the template strings are quoted properly (bash) @@ -552,6 +773,46 @@ def test_special_chars_csh(self): self.assertTrue(env_name.encode() in lines[0]) self.assertEndsWith(lines[1], env_name.encode()) + # gh-140006: the fish prompt override must keep working when a user + # function shadows a builtin it relies on. + @unittest.skipIf(os.name == 'nt', 'fish is not available on Windows') + def test_fish_activate_shadowed_builtins(self): + """ + The fish prompt override restores the exit status through `source` and + prints through `printf`/`echo`/`set_color`. A user function that + shadows one of those builtins (a common pattern for `.`-style directory + navigators) must not hijack the prompt or break status restoration. + """ + fish = shutil.which('fish') + if fish is None: + self.skipTest('fish required for this test') + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, 'activate.fish') + test_script = os.path.join(self.env_dir, 'test_shadowed_builtins.fish') + with open(test_script, "w") as f: + f.write( + # The pre-existing prompt reports the status it receives; + # activation copies it to _old_fish_prompt. + 'function fish_prompt; builtin echo "OLDSTATUS=$status"; end\n' + f'source {shlex.quote(activate)}\n' + # Shadow every builtin the override uses. A dot-navigator that + # lists the directory is the reported failure. + 'function .; builtin echo DOT_LEAK; end\n' + 'function source; builtin echo SOURCE_LEAK; end\n' + 'function echo; command echo ECHO_LEAK; end\n' + 'function printf; command printf PRINTF_LEAK; end\n' + 'function set_color; command true; end\n' + 'function _exit7; return 7; end\n' + '_exit7\n' + 'fish_prompt\n' + ) + out, err = check_output([fish, '--no-config', test_script]) + text = out.decode() + self.assertNotIn('LEAK', text) + self.assertIn('OLDSTATUS=7', text) + # gh-124651: test quoted strings on Windows @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') def test_special_chars_windows(self): @@ -592,6 +853,51 @@ def test_unicode_in_batch_file(self): ) self.assertEqual(out.strip(), '0') + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_activate_bat_respects_disable_prompt(self): + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_disable_prompt.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + 'set "PROMPT=base$G"\n' + 'set "VIRTUAL_ENV_DISABLE_PROMPT=1"\n' + f'call "{activate}"\n' + 'echo ACTIVE_PROMPT:%PROMPT%\n' + 'echo VIRTUAL_ENV:%VIRTUAL_ENV%\n' + 'set "PROMPT=changed$G"\n' + 'call deactivate\n' + 'echo FINAL_PROMPT:%PROMPT%\n') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertEqual(lines[0], b'ACTIVE_PROMPT:base$G') + self.assertEndsWith(lines[1], os.fsencode(env_dir)) + self.assertEqual(lines[2], b'FINAL_PROMPT:changed$G') + + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_activate_bat_prefixes_prompt_by_default(self): + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_enable_prompt.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + 'set "PROMPT=base) $G"\n' + 'set "VIRTUAL_ENV_DISABLE_PROMPT="\n' + f'call "{activate}"\n' + 'echo ACTIVE_PROMPT:%PROMPT%\n' + 'call deactivate\n' + 'echo FINAL_PROMPT:%PROMPT%\n') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertEqual(lines[0], b'ACTIVE_PROMPT:(venv) base) $G') + self.assertEqual(lines[1], b'FINAL_PROMPT:base) $G') + @unittest.skipUnless(os.name == 'nt' and can_symlink(), 'symlinks on Windows') def test_failed_symlink(self): @@ -656,6 +962,26 @@ def test_deactivate_with_strict_bash_opts(self): self.assertEqual(out, "".encode()) self.assertEqual(err, "".encode()) + # gh-149701: Test exit code is zero even when hashing is disabled + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + def test_deactivate_with_strict_bash_opts_and_hashing_disabled(self): + bash = shutil.which("bash") + if bash is None: + self.skipTest("bash required for this test") + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, "activate") + test_script = os.path.join(self.env_dir, "test_hash_disabled.sh") + with open(test_script, "w") as f: + f.write("set -euo pipefail\n" + "set +h\n" # disable hashing + f"source {activate}\n" + "deactivate") + out, err = check_output([bash, test_script]) + self.assertEqual(out, "".encode()) + self.assertEqual(err, "".encode()) + @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') def test_macos_env(self): @@ -694,6 +1020,12 @@ def test_zippath_from_non_installed_posix(self): os.mkdir(bindir) python_exe = os.path.basename(sys.executable) shutil.copy2(sys.executable, os.path.join(bindir, python_exe)) + if sys.platform == 'cygwin': + # Copy libpython DLL + exe_path = os.path.dirname(sys.executable) + libpython_dll = sysconfig.get_config_var('DLLLIBRARY') + shutil.copy2(os.path.join(exe_path, libpython_dll), + os.path.join(bindir, libpython_dll)) libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) os.makedirs(libdir) landmark = os.path.join(libdir, "os.py") @@ -896,10 +1228,10 @@ def test_venvwlauncher(self): exename = exename.replace("python", "pythonw") envpyw = os.path.join(self.env_dir, self.bindir, exename) try: - subprocess.check_call([envpyw, "-c", "import sys; " - "assert sys._base_executable.endswith('%s')" % exename]) + subprocess.check_call([envpyw, "-c", "import fnmatch, sys; " + "assert fnmatch.fnmatch(sys._base_executable, '**/pythonw*.exe')"]) except subprocess.CalledProcessError: - self.fail("venvwlauncher.exe did not run %s" % exename) + self.fail("venvwlauncher.exe did not run pythonw.exe") @requireVenvCreate diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index b187643e84521cc..0c4717521f16f67 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -167,6 +167,11 @@ def test_basic_callback(self): self.check_basic_callback(create_function) self.check_basic_callback(create_bound_method) + def test_non_callable_callback(self): + c = C() + self.assertRaises(TypeError, weakref.ref, c, 42) + self.assertRaises(TypeError, weakref.proxy, c, 42) + @support.cpython_only def test_cfunction(self): _testcapi = import_helper.import_module("_testcapi") diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 2ba3af8d5bf22fc..82f14ca968f266b 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -5,6 +5,7 @@ import subprocess import sys import unittest +import warnings import webbrowser from test import support from test.support import force_not_colorized_test_class @@ -335,6 +336,87 @@ def close(self): return None +@unittest.skipUnless(sys.platform == "darwin", "macOS specific test") +@requires_subprocess() +class MacOSTest(unittest.TestCase): + + def setUp(self): + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset("BROWSER") + + def test_default(self): + browser = webbrowser.get() + self.assertIsInstance(browser, webbrowser.MacOS) + self.assertEqual(browser.name, 'default') + + def test_default_http_open(self): + # http/https URLs use /usr/bin/open directly — no bundle ID needed. + browser = webbrowser.MacOS('default') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', URL], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_default_non_http_uses_bundle_id(self): + # Non-http(s) URLs (e.g. file://) must be routed through the browser + # via -b to prevent OS file handler dispatch. + file_url = 'file:///tmp/test.html' + browser = webbrowser.MacOS('default') + with mock.patch('webbrowser._macos_default_browser_bundle_id', + return_value='com.google.Chrome'), \ + mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(file_url) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-b', 'com.google.Chrome', file_url], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_named_known_browser_uses_bundle_id(self): + # Named browsers with a known bundle ID use /usr/bin/open -b. + browser = webbrowser.MacOS('safari') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-b', 'com.apple.Safari', URL], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_named_unknown_browser_falls_back_to_dash_a(self): + # Named browsers not in the bundle ID map fall back to -a. + browser = webbrowser.MacOS('lynx') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-a', 'lynx', URL], + stderr=subprocess.DEVNULL, + ) + + def test_open_failure(self): + browser = webbrowser.MacOS('default') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=1) + result = browser.open(URL) + self.assertFalse(result) + + +@unittest.skipUnless(sys.platform == "darwin", "macOS specific test") +@requires_subprocess() +class MacOSXOSAScriptDeprecationTest(unittest.TestCase): + + def test_deprecation_warning(self): + with self.assertWarns(DeprecationWarning): + webbrowser.MacOSXOSAScript('default') + + @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") @requires_subprocess() class MacOSXOSAScriptTest(unittest.TestCase): @@ -345,17 +427,14 @@ def setUp(self): env.unset("BROWSER") support.patch(self, os, "popen", self.mock_popen) + self.enterContext(warnings.catch_warnings()) + warnings.simplefilter("ignore", DeprecationWarning) self.browser = webbrowser.MacOSXOSAScript("default") def mock_popen(self, cmd, mode): self.popen_pipe = MockPopenPipe(cmd, mode) return self.popen_pipe - def test_default(self): - browser = webbrowser.get() - assert isinstance(browser, webbrowser.MacOSXOSAScript) - self.assertEqual(browser.name, "default") - def test_default_open(self): url = "https://python.org" self.browser.open(url) @@ -381,7 +460,9 @@ def test_default_browser_lookup(self): self.assertIn(f'open location "{url}"', script) def test_explicit_browser(self): - browser = webbrowser.MacOSXOSAScript("safari") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + browser = webbrowser.MacOSXOSAScript("safari") browser.open("https://python.org") script = self.popen_pipe.pipe.getvalue() self.assertIn('tell application "safari"', script) diff --git a/Lib/test/test_xml.py b/Lib/test/test_xml.py new file mode 100644 index 000000000000000..3a8b92048166f25 --- /dev/null +++ b/Lib/test/test_xml.py @@ -0,0 +1,43 @@ +import xml +import unittest + + +class TestUtils(unittest.TestCase): + + def test_is_valid_name(self): + is_valid_name = xml.is_valid_name + self.assertFalse(is_valid_name('')) + self.assertTrue(is_valid_name('name')) + self.assertTrue(is_valid_name('NAME')) + self.assertTrue(is_valid_name('name0:-._·')) + self.assertTrue(is_valid_name('_')) + self.assertTrue(is_valid_name(':')) + self.assertTrue(is_valid_name('Ñàḿĕ')) + self.assertTrue(is_valid_name('\U000EFFFF')) + self.assertFalse(is_valid_name('0')) + self.assertFalse(is_valid_name('-')) + self.assertFalse(is_valid_name('.')) + self.assertFalse(is_valid_name('·')) + self.assertFalse(is_valid_name('na me')) + for c in '<>/!?=\x00\x01\x7f\ud800\udfff\ufffe\uffff\U000F0000': + self.assertFalse(is_valid_name('name' + c)) + + def test_is_valid_text(self): + is_valid_text = xml.is_valid_text + self.assertTrue(is_valid_text('')) + self.assertTrue(is_valid_text('!0Aa_~ \r\n\t\x85\xa0')) + self.assertTrue(is_valid_text('\ud7ff\ue000\ufffd\U00010000\U0010ffff')) + self.assertFalse(is_valid_text('\x00')) + self.assertFalse(is_valid_text('\x01')) + self.assertFalse(is_valid_text('\x1f')) + self.assertTrue(is_valid_text('\x7f')) + self.assertTrue(is_valid_text('\x80')) + self.assertTrue(is_valid_text('\x9f')) + self.assertFalse(is_valid_text('\ud800')) + self.assertFalse(is_valid_text('\udfff')) + self.assertFalse(is_valid_text('\ufffe')) + self.assertFalse(is_valid_text('\uffff')) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 8f3efe9fc90794b..acec4ec2ca257c4 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -1008,13 +1008,15 @@ def check(encoding, body=''): check("iso-8859-15", '\u20ac') check("cp437", '\u221a') check("mac-roman", '\u02da') + check('shift-jis-2004', '\u203e\u3406\uff66') + check('euc-jis-2004', '\u3406\uff66') - def xml(encoding): - return "" % encoding - def bxml(encoding): - return xml(encoding).encode(encoding) + def xml(encoding, body=''): + return "%s" % (encoding, body) + def bxml(encoding, body=''): + return xml(encoding, body).encode(encoding) supported_encodings = [ - 'ascii', 'utf-8', 'utf-8-sig', 'utf-16', 'utf-16be', 'utf-16le', + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', @@ -1025,36 +1027,54 @@ def bxml(encoding): 'cp1256', 'cp1257', 'cp1258', 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', - 'iso2022-jp', 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', - 'iso2022-jp-3', 'iso2022-jp-ext', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', - 'hz', 'ptcp154', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', + 'big5', 'big5hkscs', + 'cp932', 'cp949', 'cp950', + 'euc-jp', 'euc-jis-2004', 'euc-jisx0213', + 'gb2312', 'gbk', 'johab', + 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', + 'utf-8-sig', 'utf8', ] for encoding in supported_encodings: - self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + with self.subTest(encoding=encoding): + self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'') + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + self.assertEqual(ET.tostring(ET.XML(bxml(encoding, c))), + ('&#%d;' % ord(c)).encode()) unsupported_ascii_compatible_encodings = [ - 'big5', 'big5hkscs', - 'cp932', 'cp949', 'cp950', - 'euc-jp', 'euc-jis-2004', 'euc-jisx0213', 'euc-kr', - 'gb2312', 'gbk', 'gb18030', - 'iso2022-kr', 'johab', - 'shift-jis', 'shift-jis-2004', 'shift-jisx0213', + 'euc-kr', 'gb18030', + 'iso2022-jp', 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', + 'iso2022-jp-3', 'iso2022-jp-ext', + 'iso2022-kr', 'hz', 'utf-7', ] for encoding in unsupported_ascii_compatible_encodings: - self.assertRaises(ValueError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ValueError, ET.XML, bxml(encoding)) unsupported_ascii_incompatible_encodings = [ 'cp037', 'cp424', 'cp500', 'cp864', 'cp875', 'cp1026', 'cp1140', 'utf_32', 'utf_32_be', 'utf_32_le', ] for encoding in unsupported_ascii_incompatible_encodings: - self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) + @support.subTests('sample,exception', [ + (b' \xa1', UnicodeDecodeError), # crashed + (b' \xa1 \xa1', None), # ET.ParseError + ]) + def test_multibyte_encoding_errors(self, sample, exception): + exception = exception or ET.ParseError + data = b'\n' + sample + with self.assertRaises(exception): + ET.XML(data) + def test_methods(self): # Test serialization methods. @@ -1278,7 +1298,15 @@ def check(p, expected, namespaces=None): {'': 'http://www.w3.org/2001/XMLSchema', 'ns': 'http://www.w3.org/2001/XMLSchema'}) - def test_processinginstruction(self): + def test_comment_serialization(self): + comm = ET.Comment(' & ham') + # comments are not escaped + self.assertEqual(ET.tostring(comm), b'') + self.assertEqual(ET.tostring(comm, method='html'), b'') + # no comments in text serialization + self.assertEqual(ET.tostring(comm, method='text'), b'') + + def test_processinginstruction_serialization(self): # Test ProcessingInstruction directly self.assertEqual(ET.tostring(ET.ProcessingInstruction('test', 'instruction')), @@ -1287,12 +1315,32 @@ def test_processinginstruction(self): b'') # Issue #2746 - + # processing instructions are not escaped self.assertEqual(ET.tostring(ET.PI('test', '')), b'?>') self.assertEqual(ET.tostring(ET.PI('test', '\xe3'), 'latin-1'), b"\n" b"\xe3?>") + pi = ET.PI('test', 'ham & eggs < spam') + self.assertEqual(ET.tostring(pi), b'') + self.assertEqual(ET.tostring(pi, method='html'), b'') + # no processing instructions in text serialization + self.assertEqual(ET.tostring(pi, method='text'), b'') + + def test_empty_attribute_serialization(self): + # empty attrs only work in html + elem = ET.Element('tag', attrib={'attr': None}) + self.assertRaises(TypeError, ET.tostring, elem) + self.assertEqual(ET.tostring(elem, method='html'), b'') + + @support.subTests('tag', ("script", "style", "xmp", "iframe", "noembed", "noframes")) + def test_html_cdata_elems_serialization(self, tag): + # content of raw text elements is not escaped in html + tag = tag.title() + elem = ET.Element(tag) + elem.text = '&ham' + self.assertEqual(ET.tostring(elem, method='html'), + ('<%s>&ham' % (tag, tag)).encode()) def test_html_empty_elems_serialization(self): # issue 15970 @@ -1308,6 +1356,14 @@ def test_html_empty_elems_serialization(self): method='html') self.assertEqual(serialized, expected) + def test_html_plaintext_serialization(self): + # content of plaintext is not escaped in html + # no end tag for plaintext + elem = ET.Element('PlainText') + elem.text = '&ham' + self.assertEqual(ET.tostring(elem, method='html'), + b'<spam>&ham') + def test_dump_attribute_order(self): # See BPO 34160 e = ET.Element('cirriculum', status='public', company='example') @@ -3654,6 +3710,32 @@ def test_basic(self): doc = ET.XML("<root>a&amp;<sub>b&amp;</sub>c&amp;</root>") self.assertEqual(''.join(doc.itertext()), 'a&b&c&') + def test_comment(self): + e = ET.Element('root') + e.text = 'before' + comment = ET.Comment('comment') + self.assertEqual(comment.text, 'comment') + comment.tail = 'after' + e.append(comment) + self.assertEqual(''.join(e.itertext()), 'beforeafter') + self.assertEqual(list(e.iter()), [e, comment]) + self.assertEqual(list(e.iter('root')), [e]) + self.assertEqual(''.join(comment.itertext()), '') + self.assertEqual(list(comment.iter()), [comment]) + + def test_processinginstruction(self): + e = ET.Element('root') + e.text = 'before' + pi = ET.PI('test', 'instruction') + self.assertEqual(pi.text, 'test instruction') + pi.tail = 'after' + e.append(pi) + self.assertEqual(''.join(e.itertext()), 'beforeafter') + self.assertEqual(list(e.iter()), [e, pi]) + self.assertEqual(list(e.iter('root')), [e]) + self.assertEqual(''.join(pi.itertext()), '') + self.assertEqual(list(pi.iter()), [pi]) + def test_corners(self): # single root, no subelements a = ET.Element('a') diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 2803c6d45c27bfa..ee0e24f6e86ae33 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -208,6 +208,17 @@ def test_dump_encoding(self): self.assertEqual(xmlrpclib.loads(strg)[0][0], value) self.assertEqual(xmlrpclib.loads(strg)[1], methodname) + def test_dump_escape_methodname(self): + payload = 'foo</methodName><injected attr="evil"/><methodName>bar' + s = xmlrpclib.dumps((), methodname=payload) + self.assertIn( + '<methodName>foo&lt;/methodName&gt;&lt;injected attr="evil"/&gt;' + '&lt;methodName&gt;bar</methodName>', s + ) + self.assertNotIn('<injected attr="evil"/>', s) + load, m = xmlrpclib.loads(s) + self.assertEqual(m, payload) + def test_dump_bytes(self): sample = b"my dog has fleas" self.assertEqual(sample, xmlrpclib.Binary(sample)) diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py index b52e78bc4fb7e05..c6e9dc375d9a676 100644 --- a/Lib/test/test_xxlimited.py +++ b/Lib/test/test_xxlimited.py @@ -1,19 +1,39 @@ import unittest from test.support import import_helper -import types xxlimited = import_helper.import_module('xxlimited') -xxlimited_35 = import_helper.import_module('xxlimited_35') - -class CommonTests: - module: types.ModuleType - - def test_xxo_new(self): - xxo = self.module.Xxo() - - def test_xxo_attributes(self): - xxo = self.module.Xxo() +# if import of xxlimited succeeded, the other ones should be importable. +import xxlimited_3_13 +import xxlimited_35 + +MODULES = { + (3, 15): xxlimited, + (3, 13): xxlimited_3_13, + (3, 5): xxlimited_35, +} + +def test_with_xxlimited_modules(since=None, until=None): + def _decorator(func): + def _wrapper(self, *args, **kwargs): + for version, module in MODULES.items(): + if since and version < since: + continue + if until and version >= until: + continue + with self.subTest(version=version): + func(self, module, *args, **kwargs) + return _wrapper + return _decorator + +class XXLimitedTests(unittest.TestCase): + @test_with_xxlimited_modules() + def test_xxo_new(self, module): + xxo = module.Xxo() + + @test_with_xxlimited_modules() + def test_xxo_attributes(self, module): + xxo = module.Xxo() with self.assertRaises(AttributeError): xxo.foo with self.assertRaises(AttributeError): @@ -26,40 +46,61 @@ def test_xxo_attributes(self): with self.assertRaises(AttributeError): xxo.foo - def test_foo(self): + @test_with_xxlimited_modules() + def test_foo(self, module): # the foo function adds 2 numbers - self.assertEqual(self.module.foo(1, 2), 3) + self.assertEqual(module.foo(1, 2), 3) - def test_str(self): - self.assertIsSubclass(self.module.Str, str) - self.assertIsNot(self.module.Str, str) + @test_with_xxlimited_modules() + def test_str(self, module): + self.assertIsSubclass(module.Str, str) + self.assertIsNot(module.Str, str) - custom_string = self.module.Str("abcd") + custom_string = module.Str("abcd") self.assertEqual(custom_string, "abcd") self.assertEqual(custom_string.upper(), "ABCD") - def test_new(self): - xxo = self.module.new() + @test_with_xxlimited_modules() + def test_new(self, module): + xxo = module.new() self.assertEqual(xxo.demo("abc"), "abc") - -class TestXXLimited(CommonTests, unittest.TestCase): - module = xxlimited - - def test_xxo_demo(self): - xxo = self.module.Xxo() - other = self.module.Xxo() + @test_with_xxlimited_modules() + def test_xxo_demo(self, module): + xxo = module.Xxo() self.assertEqual(xxo.demo("abc"), "abc") + self.assertEqual(xxo.demo(0), None) + self.assertEqual(xxo.__module__, module.__name__) + with self.assertRaises(TypeError): + module.Xxo('arg') + with self.assertRaises(TypeError): + module.Xxo(kwarg='arg') + + @test_with_xxlimited_modules(since=(3, 13)) + def test_xxo_demo_extra(self, module): + xxo = module.Xxo() + other = module.Xxo() self.assertEqual(xxo.demo(xxo), xxo) self.assertEqual(xxo.demo(other), other) - self.assertEqual(xxo.demo(0), None) - def test_error(self): - with self.assertRaises(self.module.Error): - raise self.module.Error - - def test_buffer(self): - xxo = self.module.Xxo() + @test_with_xxlimited_modules(since=(3, 15)) + def test_xxo_subclass(self, module): + class Sub(module.Xxo): + pass + sub = Sub() + sub.a = 123 + self.assertEqual(sub.a, 123) + with self.assertRaisesRegex(AttributeError, "cannot set 'reserved'"): + sub.reserved = 123 + + @test_with_xxlimited_modules(since=(3, 13)) + def test_error(self, module): + with self.assertRaises(module.Error): + raise module.Error + + @test_with_xxlimited_modules(since=(3, 13)) + def test_buffer(self, module): + xxo = module.Xxo() self.assertEqual(xxo.x_exports, 0) b1 = memoryview(xxo) self.assertEqual(xxo.x_exports, 1) @@ -69,21 +110,13 @@ def test_buffer(self): self.assertEqual(b1[0], 1) self.assertEqual(b2[0], 1) - -class TestXXLimited35(CommonTests, unittest.TestCase): - module = xxlimited_35 - - def test_xxo_demo(self): - xxo = self.module.Xxo() - other = self.module.Xxo() - self.assertEqual(xxo.demo("abc"), "abc") - self.assertEqual(xxo.demo(0), None) - + @test_with_xxlimited_modules(until=(3, 5)) def test_roj(self): # the roj function always fails with self.assertRaises(SystemError): self.module.roj(0) + @test_with_xxlimited_modules(until=(3, 5)) def test_null(self): null1 = self.module.Null() null2 = self.module.Null() diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 6887a5e5cc4d18b..4f20209927e7b3d 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -13,6 +13,7 @@ import time import unittest import unittest.mock as mock +import warnings import zipfile @@ -1434,6 +1435,2125 @@ class LzmaWriterTests(AbstractWriterTests, unittest.TestCase): class ZstdWriterTests(AbstractWriterTests, unittest.TestCase): compression = zipfile.ZIP_ZSTANDARD + +_ZINFO_PUBLIC_KEYS = [k for k in zipfile.ZipInfo.__slots__ if not k.startswith('_')] + +def comparable_zinfo(zinfo): + """Return a dict of public ZipInfo attributes for assertEqual comparison.""" + attrs = {k: getattr(zinfo, k) for k in _ZINFO_PUBLIC_KEYS} + + # Since patch gh-84353, the _MASK_UTF_FILENAME (0x800) bit may be + # changed when writing to the end record depending on whether filename + # can be encoded with ascii or cp437. Skip checking this bit by + # pretending it's always set. + attrs['flag_bits'] |= 0x800 + + return attrs + +_struct_pack = struct.pack + +def struct_pack_no_dd_sig(fmt, *values): + """A mock side_effect for native `struct.pack` to not generate a + signature for data descriptors.""" + # suppress BytesWarning etc. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + if values[0] == zipfile._DD_SIGNATURE: + return _struct_pack(fmt[:1] + fmt[2:], *values[1:]) + return _struct_pack(fmt, *values) + +class RepackHelperMixin: + """Common helpers for remove and repack.""" + maxDiff = 8192 + + @classmethod + def _prepare_test_files(cls): + return [ + ('file0.txt', b'Lorem ipsum dolor sit amet, consectetur adipiscing elit'), + ('file1.txt', b'Duis aute irure dolor in reprehenderit in voluptate velit esse'), + ('file2.txt', b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem'), + ] + + @classmethod + def _prepare_zip_from_test_files(cls, zfname, test_files, force_zip64=False): + with zipfile.ZipFile(zfname, 'w', cls.compression) as zh: + for file, data in test_files: + with zh.open(file, 'w', force_zip64=force_zip64) as fh: + fh.write(data) + return list(zh.infolist()) + +class AbstractRemoveTests(RepackHelperMixin): + @classmethod + def setUpClass(cls): + cls.test_files = cls._prepare_test_files() + + def tearDown(self): + unlink(TESTFN) + + def test_remove_by_name(self): + for i in range(0, 3): + with self.subTest(i=i, filename=self.test_files[i][0]): + zinfos = self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zh.remove(self.test_files[i][0]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for j, zi in enumerate(zinfos) if j != i], + ) + + # check NameToInfo cache + with self.assertRaises(KeyError): + zh.getinfo(self.test_files[i][0]) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_remove_by_zinfo(self): + for i in range(0, 3): + with self.subTest(i=i, filename=self.test_files[i][0]): + zinfos = self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zh.remove(zh.infolist()[i]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for j, zi in enumerate(zinfos) if j != i], + ) + + # check NameToInfo cache + with self.assertRaises(KeyError): + zh.getinfo(self.test_files[i][0]) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_remove_by_name_nonexist(self): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + with self.assertRaises(KeyError): + zh.remove('nonexist.txt') + + def test_remove_by_zinfo_nonexist(self): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + with self.assertRaises(KeyError): + zh.remove(zipfile.ZipInfo('nonexist.txt')) + + def test_remove_by_name_duplicated(self): + test_files = [ + ('file.txt', b'Lorem ipsum dolor sit amet, consectetur adipiscing elit'), + ('file.txt', b'Duis aute irure dolor in reprehenderit in voluptate velit esse'), + ('file1.txt', b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem'), + ] + + # suppress duplicated name warning + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + zinfos = self._prepare_zip_from_test_files(TESTFN, test_files) + + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zh.remove('file.txt') + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in [zinfos[0], zinfos[2]]], + ) + + # check NameToInfo cache + self.assertEqual( + comparable_zinfo(zh.getinfo('file.txt')), + comparable_zinfo(zinfos[0]), + ) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + # suppress duplicated name warning + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + zinfos = self._prepare_zip_from_test_files(TESTFN, test_files) + + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zh.remove('file.txt') + zh.remove('file.txt') + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in [zinfos[2]]], + ) + + # check NameToInfo cache + with self.assertRaises(KeyError): + zh.getinfo('file.txt') + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_remove_by_zinfo_duplicated(self): + test_files = [ + ('file.txt', b'Lorem ipsum dolor sit amet, consectetur adipiscing elit'), + ('file.txt', b'Duis aute irure dolor in reprehenderit in voluptate velit esse'), + ('file1.txt', b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem'), + ] + + # suppress duplicated name warning + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + zinfos = self._prepare_zip_from_test_files(TESTFN, test_files) + + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zh.remove(zh.infolist()[0]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in [zinfos[1], zinfos[2]]], + ) + + # check NameToInfo cache + self.assertEqual( + comparable_zinfo(zh.getinfo('file.txt')), + comparable_zinfo(zinfos[1]), + ) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + # suppress duplicated name warning + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + zinfos = self._prepare_zip_from_test_files(TESTFN, test_files) + + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zh.remove(zh.infolist()[1]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in [zinfos[0], zinfos[2]]], + ) + + # check NameToInfo cache + self.assertEqual( + comparable_zinfo(zh.getinfo('file.txt')), + comparable_zinfo(zinfos[0]), + ) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + # suppress duplicated name warning + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + zinfos = self._prepare_zip_from_test_files(TESTFN, test_files) + + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + infolist = zh.infolist().copy() + zh.remove(infolist[0]) + zh.remove(infolist[1]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in [zinfos[2]]], + ) + + # check NameToInfo cache + with self.assertRaises(KeyError): + zh.getinfo('file.txt') + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_remove_zip64(self): + for i in range(0, 3): + with self.subTest(i=i, filename=self.test_files[i][0]): + zinfos = self._prepare_zip_from_test_files(TESTFN, self.test_files, force_zip64=True) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zh.remove(zh.infolist()[i]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for j, zi in enumerate(zinfos) if j != i], + ) + + # check NameToInfo cache + with self.assertRaises(KeyError): + zh.getinfo(self.test_files[i][0]) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_remove_closed(self): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + zh.close() + with self.assertRaises(ValueError): + zh.remove(self.test_files[0][0]) + + def test_remove_writing(self): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + with zh.open('newfile.txt', 'w'): + with self.assertRaises(ValueError): + zh.remove(self.test_files[0][0]) + + def test_remove_mode_r(self): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'r') as zh: + with self.assertRaises(ValueError): + zh.remove(self.test_files[0][0]) + + def test_remove_mode_w(self): + with zipfile.ZipFile(TESTFN, 'w') as zh: + for file, data in self.test_files: + zh.writestr(file, data) + zinfos = list(zh.infolist()) + + zh.remove(self.test_files[0][0]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in [zinfos[1], zinfos[2]]], + ) + + # check NameToInfo cache + with self.assertRaises(KeyError): + zh.getinfo(self.test_files[0][0]) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_remove_mode_x(self): + with zipfile.ZipFile(TESTFN, 'x') as zh: + for file, data in self.test_files: + zh.writestr(file, data) + zinfos = list(zh.infolist()) + + zh.remove(self.test_files[0][0]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in [zinfos[1], zinfos[2]]], + ) + + # check NameToInfo cache + with self.assertRaises(KeyError): + zh.getinfo(self.test_files[0][0]) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + +class StoredRemoveTests(AbstractRemoveTests, unittest.TestCase): + compression = zipfile.ZIP_STORED + +@requires_zlib() +class DeflateRemoveTests(AbstractRemoveTests, unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2() +class Bzip2RemoveTests(AbstractRemoveTests, unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma() +class LzmaRemoveTests(AbstractRemoveTests, unittest.TestCase): + compression = zipfile.ZIP_LZMA + +@requires_zstd() +class ZstdRemoveTests(AbstractRemoveTests, unittest.TestCase): + compression = zipfile.ZIP_ZSTANDARD + +class AbstractRepackTests(RepackHelperMixin): + @classmethod + def setUpClass(cls): + cls.test_files = cls._prepare_test_files() + + def tearDown(self): + unlink(TESTFN) + + def test_repack_basic(self): + """Should remove local file entries for deleted files.""" + ln = len(self.test_files) + iii = (ii for n in range(1, ln + 1) for ii in itertools.combinations(range(ln), n)) + for ii in iii: + with self.subTest(remove=ii): + # calculate the expected results + test_files = [data for j, data in enumerate(self.test_files) if j not in ii] + expected_zinfos = self._prepare_zip_from_test_files(TESTFN, test_files) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + for i in ii: + zh.remove(self.test_files[i][0]) + zh.repack() + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_repack_propagation(self): + """Should call internal API with adequate parameters.""" + self._prepare_zip_from_test_files(TESTFN, self.test_files) + + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + with mock.patch.object(zipfile._ZipRepacker, 'repack') as m_rp, \ + mock.patch.object(zipfile, '_ZipRepacker', wraps=zipfile._ZipRepacker) as m_zr: + zh.repack() + m_zr.assert_called_once_with(strict_descriptor=True, + chunk_size=zipfile._REPACK_CHUNK_SIZE) + m_rp.assert_called_once_with(zh, None) + + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zi = zh.remove(zh.infolist()[0]) + with mock.patch.object(zipfile._ZipRepacker, 'repack') as m_rp, \ + mock.patch.object(zipfile, '_ZipRepacker', wraps=zipfile._ZipRepacker) as m_zr: + zh.repack([zi], strict_descriptor=False, chunk_size=1024) + m_zr.assert_called_once_with(strict_descriptor=False, chunk_size=1024) + m_rp.assert_called_once_with(zh, [zi]) + + def test_repack_bytes_before_first_file(self): + """Should preserve random bytes before the first recorded local file entry.""" + for ii in ([], [0], [0, 1], [0, 1, 2]): + with self.subTest(remove=ii): + # calculate the expected results + test_files = [data for j, data in enumerate(self.test_files) if j not in ii] + with open(TESTFN, 'wb') as fh: + fh.write(b'dummy ') + expected_zinfos = self._prepare_zip_from_test_files(fh, test_files) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + with open(TESTFN, 'wb') as fh: + fh.write(b'dummy ') + self._prepare_zip_from_test_files(fh, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + for i in ii: + zh.remove(self.test_files[i][0]) + zh.repack() + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_repack_magic_before_first_file(self): + """Should preserve random signature bytes not forming a valid file entry + before the first recorded local file entry.""" + for ii in ([], [0], [0, 1], [0, 1, 2]): + with self.subTest(remove=ii): + # calculate the expected results + test_files = [data for j, data in enumerate(self.test_files) if j not in ii] + with open(TESTFN, 'wb') as fh: + fh.write(b'PK\003\004 ') + expected_zinfos = self._prepare_zip_from_test_files(fh, test_files) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + with open(TESTFN, 'wb') as fh: + fh.write(b'PK\003\004 ') + self._prepare_zip_from_test_files(fh, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + for i in ii: + zh.remove(self.test_files[i][0]) + zh.repack() + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_repack_file_entry_before_first_file(self): + """Should preserve seemingly valid file entries not forming consecutive + valid file entries until the first recorded local file entry. + + This may happen when a self-extractor contains an uncompressed ZIP + library. (simulated by writing a ZIP file in this test) + """ + for ii in ([], [0], [0, 1], [0, 1, 2]): + with self.subTest(remove=ii): + # calculate the expected results + test_files = [data for j, data in enumerate(self.test_files) if j not in ii] + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w') as zh: + zh.writestr('file.txt', b'dummy') + zh.writestr('file2.txt', b'dummy') + zh.writestr('file3.txt', b'dummy') + fh.write(b' ') + expected_zinfos = self._prepare_zip_from_test_files(fh, test_files) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w') as zh: + zh.writestr('file.txt', b'dummy') + zh.writestr('file2.txt', b'dummy') + zh.writestr('file3.txt', b'dummy') + fh.write(b' ') + self._prepare_zip_from_test_files(fh, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + for i in ii: + zh.remove(self.test_files[i][0]) + zh.repack() + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + @mock.patch.object(time, 'time', new=lambda: 315590400) # fix time for ZipFile.writestr() + def test_repack_bytes_before_removed_files(self): + """Should preserve if there are bytes before stale local file entries.""" + for ii in ([1], [1, 2], [2]): + with self.subTest(remove=ii): + # calculate the expected results + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w', self.compression) as zh: + for i, (file, data) in enumerate(self.test_files): + if i == ii[0]: + fh.write(b' dummy bytes ') + zh.start_dir = fh.tell() + zh.writestr(file, data) + for i in ii: + zh.remove(self.test_files[i][0]) + expected_zinfos = list(zh.infolist()) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w', self.compression) as zh: + for i, (file, data) in enumerate(self.test_files): + if i == ii[0]: + fh.write(b' dummy bytes ') + zh.start_dir = fh.tell() + zh.writestr(file, data) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + for i in ii: + zh.remove(self.test_files[i][0]) + zh.repack() + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + @mock.patch.object(time, 'time', new=lambda: 315590400) # fix time for ZipFile.writestr() + def test_repack_bytes_after_removed_files(self): + """Should keep extra bytes if there are bytes after stale local file entries.""" + for ii in ([1], [1, 2], [2]): + with self.subTest(remove=ii): + # calculate the expected results + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w', self.compression) as zh: + for i, (file, data) in enumerate(self.test_files): + if i not in ii: + zh.writestr(file, data) + if i == ii[-1]: + fh.write(b' dummy bytes ') + zh.start_dir = fh.tell() + expected_zinfos = list(zh.infolist()) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w', self.compression) as zh: + for i, (file, data) in enumerate(self.test_files): + zh.writestr(file, data) + if i == ii[-1]: + fh.write(b' dummy bytes ') + zh.start_dir = fh.tell() + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + for i in ii: + zh.remove(self.test_files[i][0]) + zh.repack() + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + @mock.patch.object(time, 'time', new=lambda: 315590400) # fix time for ZipFile.writestr() + def test_repack_bytes_between_removed_files(self): + """Should strip only local file entries before random bytes.""" + # calculate the expected results + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w', self.compression) as zh: + zh.writestr(*self.test_files[0]) + fh.write(b' dummy bytes ') + zh.start_dir = fh.tell() + zh.writestr(*self.test_files[2]) + zh.remove(self.test_files[2][0]) + expected_zinfos = list(zh.infolist()) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w', self.compression) as zh: + zh.writestr(*self.test_files[0]) + zh.writestr(*self.test_files[1]) + fh.write(b' dummy bytes ') + zh.start_dir = fh.tell() + zh.writestr(*self.test_files[2]) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zh.remove(self.test_files[1][0]) + zh.remove(self.test_files[2][0]) + zh.repack() + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_repack_prepended_bytes(self): + for ii in ([], [0], [0, 1], [1], [2]): + with self.subTest(remove=ii): + # calculate the expected results + fz = io.BytesIO() + test_files = [data for j, data in enumerate(self.test_files) if j not in ii] + self._prepare_zip_from_test_files(fz, test_files) + fz.seek(0) + with open(TESTFN, 'wb') as fh: + fh.write(b'dummy ') + fh.write(fz.read()) + with zipfile.ZipFile(TESTFN) as zh: + expected_zinfos = list(zh.infolist()) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + fz = io.BytesIO() + self._prepare_zip_from_test_files(fz, self.test_files) + fz.seek(0) + with open(TESTFN, 'wb') as fh: + fh.write(b'dummy ') + fh.write(fz.read()) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + for i in ii: + zh.remove(self.test_files[i][0]) + zh.repack() + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_repack_overlapping_blocks(self): + for ii in ([0], [1], [2]): + with self.subTest(remove=ii): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + zh._didModify = True + for i in ii: + zi = zh.infolist()[i] + zi.compress_size += 1 + zi.file_size += 1 + + with zipfile.ZipFile(TESTFN, 'a') as zh: + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack() + + def test_repack_scan_unsigned_data_descriptor(self): + """By default (strict_descriptor=True) the scan does not reclaim an + unreferenced entry written with an unsigned data descriptor, but keeps + the archive valid; strict_descriptor=False reclaims it.""" + removed_name, removed_data = self.test_files[1] + remaining = [n for n, _ in self.test_files if n != removed_name] + + # Build an archive whose entries use *unsigned* data descriptors by + # writing to an unseekable stream with the descriptor signature stripped. + buf = io.BytesIO() + with mock.patch.object(struct, 'pack', side_effect=struct_pack_no_dd_sig): + with zipfile.ZipFile(Unseekable(buf), 'w', self.compression) as zh: + for file, data in self.test_files: + with zh.open(file, 'w') as fh: + fh.write(data) + archive = buf.getvalue() + + # sanity: the removed entry really uses a data descriptor (flag bit 3); + # it is unsigned by construction above + with zipfile.ZipFile(io.BytesIO(archive)) as zh: + self.assertTrue(zh.getinfo(removed_name).flag_bits & 0x08) + + # default repack(): strict_descriptor=True does not locate the unsigned + # data descriptor, so the local data is preserved (not reclaimed). + fz = io.BytesIO(archive) + with zipfile.ZipFile(fz, 'a', self.compression) as zh: + zh.remove(removed_name) + zh.repack() + default_size = len(fz.getvalue()) + with zipfile.ZipFile(fz) as zh: + self.assertEqual(zh.namelist(), remaining) + self.assertIsNone(zh.testzip()) + + # strict_descriptor=False: the unsigned data descriptor is detected, so + # the local data is reclaimed and the archive shrinks. + fz = io.BytesIO(archive) + with zipfile.ZipFile(fz, 'a', self.compression) as zh: + zh.remove(removed_name) + zh.repack(strict_descriptor=False) + strict_false_size = len(fz.getvalue()) + with zipfile.ZipFile(fz) as zh: + self.assertEqual(zh.namelist(), remaining) + self.assertIsNone(zh.testzip()) + + self.assertLess(strict_false_size, default_size) + + def test_repack_removed_basic(self): + """Should remove local file entries for provided deleted files.""" + ln = len(self.test_files) + iii = (ii for n in range(1, ln + 1) for ii in itertools.combinations(range(ln), n)) + for ii in iii: + with self.subTest(remove=ii): + # calculate the expected results + test_files = [data for j, data in enumerate(self.test_files) if j not in ii] + expected_zinfos = self._prepare_zip_from_test_files(TESTFN, test_files) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + zinfos = self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zinfos = [zh.remove(self.test_files[i][0]) for i in ii] + zh.repack(zinfos) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_repack_removed_partial(self): + """Should remove local file entries only for provided deleted files.""" + ln = len(self.test_files) + iii = (ii for n in range(1, ln + 1) for ii in itertools.combinations(range(ln), n)) + for ii in iii: + with self.subTest(removed=ii): + # calculate the expected results + test_files = [data for j, data in enumerate(self.test_files) if j not in ii] + self._prepare_zip_from_test_files(TESTFN, test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + for zi in zh.infolist().copy(): + zh.remove(zi) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + zinfos = self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zinfos = [zh.remove(self.test_files[i][0]) for i, _ in enumerate(self.test_files)] + zh.repack([zinfos[i] for i in ii]) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + @mock.patch.object(time, 'time', new=lambda: 315590400) # fix time for ZipFile.writestr() + def test_repack_removed_bytes_between_files(self): + """Should not remove bytes between local file entries.""" + for ii in ([0], [1], [2]): + with self.subTest(removed=ii): + # calculate the expected results + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w', self.compression) as zh: + for j, (file, data) in enumerate(self.test_files): + if j not in ii: + zh.writestr(file, data) + fh.write(b' dummy bytes ') + zh.start_dir = fh.tell() + expected_zinfos = list(zh.infolist()) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + with open(TESTFN, 'wb') as fh: + with zipfile.ZipFile(fh, 'w', self.compression) as zh: + for file, data in self.test_files: + zh.writestr(file, data) + fh.write(b' dummy bytes ') + zh.start_dir = fh.tell() + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zinfos = [zh.remove(self.test_files[i][0]) for i in ii] + zh.repack(zinfos) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + def test_repack_removed_bad_header_offset(self): + """Should raise when provided ZipInfo objects has differing header offset.""" + for ii in ([0], [1], [2]): + with self.subTest(removed=ii): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + zinfos = [zh.remove(self.test_files[i][0]) for i in ii] + for zi in zinfos: + zi.header_offset += 1 + with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number for file header'): + zh.repack(zinfos) + + def test_repack_removed_bad_header_offset2(self): + """Should raise when provided ZipInfo objects has differing header offset.""" + for ii in ([1], [2]): + with self.subTest(removed=ii): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + zinfos = [zh.remove(self.test_files[i][0]) for i in ii] + for zi in zinfos: + zi.header_offset -= 1 + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack(zinfos) + + def test_repack_removed_bad_non_removed(self): + """Should raise when provided ZipInfo objects are not removed.""" + for ii in ([0], [1], [2]): + with self.subTest(removed=ii): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + zinfos = [zh.getinfo(self.test_files[i][0]) for i in ii] + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack(zinfos) + + def test_repack_removed_prepended_bytes(self): + for ii in ([], [0], [0, 1], [1], [2]): + with self.subTest(remove=ii): + # calculate the expected results + test_files = [data for j, data in enumerate(self.test_files) if j not in ii] + fz = io.BytesIO() + self._prepare_zip_from_test_files(fz, test_files) + fz.seek(0) + with open(TESTFN, 'wb') as fh: + fh.write(b'dummy ') + fh.write(fz.read()) + with zipfile.ZipFile(TESTFN) as zh: + expected_zinfos = list(zh.infolist()) + expected_size = os.path.getsize(TESTFN) + + # do the removal and check the result + fz = io.BytesIO() + self._prepare_zip_from_test_files(fz, self.test_files) + fz.seek(0) + with open(TESTFN, 'wb') as fh: + fh.write(b'dummy ') + fh.write(fz.read()) + with zipfile.ZipFile(TESTFN, 'a', self.compression) as zh: + zinfos = [zh.remove(self.test_files[i][0]) for i in ii] + zh.repack(zinfos) + + # check infolist + self.assertEqual( + [comparable_zinfo(zi) for zi in zh.infolist()], + [comparable_zinfo(zi) for zi in expected_zinfos], + ) + + # check file size + self.assertEqual(os.path.getsize(TESTFN), expected_size) + + # make sure the zip file is still valid + with zipfile.ZipFile(TESTFN) as zh: + self.assertIsNone(zh.testzip()) + + @mock.patch.object(zipfile, '_ZipRepacker') + def test_repack_closed(self, m_repack): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + zh.close() + with self.assertRaises(ValueError): + zh.repack() + m_repack.assert_not_called() + + @mock.patch.object(zipfile, '_ZipRepacker') + def test_repack_writing(self, m_repack): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'a') as zh: + with zh.open('newfile.txt', 'w'): + with self.assertRaises(ValueError): + zh.repack() + m_repack.assert_not_called() + + @mock.patch.object(zipfile, '_ZipRepacker') + def test_repack_mode_r(self, m_repack): + self._prepare_zip_from_test_files(TESTFN, self.test_files) + with zipfile.ZipFile(TESTFN, 'r') as zh: + with self.assertRaises(ValueError): + zh.repack() + m_repack.assert_not_called() + + @mock.patch.object(zipfile, '_ZipRepacker') + def test_repack_mode_w(self, m_repack): + with zipfile.ZipFile(TESTFN, 'w') as zh: + with self.assertRaises(ValueError): + zh.repack() + m_repack.assert_not_called() + + @mock.patch.object(zipfile, '_ZipRepacker') + def test_repack_mode_x(self, m_repack): + with zipfile.ZipFile(TESTFN, 'x') as zh: + with self.assertRaises(ValueError): + zh.repack() + m_repack.assert_not_called() + +class StoredRepackTests(AbstractRepackTests, unittest.TestCase): + compression = zipfile.ZIP_STORED + +@requires_zlib() +class DeflateRepackTests(AbstractRepackTests, unittest.TestCase): + compression = zipfile.ZIP_DEFLATED + +@requires_bz2() +class Bzip2RepackTests(AbstractRepackTests, unittest.TestCase): + compression = zipfile.ZIP_BZIP2 + +@requires_lzma() +class LzmaRepackTests(AbstractRepackTests, unittest.TestCase): + compression = zipfile.ZIP_LZMA + +@requires_zstd() +class ZstdRepackTests(AbstractRepackTests, unittest.TestCase): + compression = zipfile.ZIP_ZSTANDARD + +class OtherRepackTests(unittest.TestCase): + def test_full_overlap_different_names(self): + # see `test_full_overlap_different_names` in built-in test.test_zipfile + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00b\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00aPK' + b'\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00bPK\x05' + b'\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00/\x00\x00' + b'\x00\x00\x00' + ) + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack() + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + zi = zh.remove('a') + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack([zi]) + + # local entry of 'a' should not be stripped (not found) + fz = io.BytesIO(data) + with zipfile.ZipFile(fz, 'a') as zh: + zh.remove('a') + zh.repack() + + expected = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' + b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00b\xed' + b'\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\d\x0b`P' + b'K\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2' + b'\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00b' + b'PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00/\x00' + b'\x00\x00\x00\x00' + ) + fz.seek(0) + self.assertEqual(fz.read(), expected) + + def test_quoted_overlap(self): + # see `test_quoted_overlap` in built-in test.test_zipfile + data = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' + b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' + b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\' + b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0' + b'lH\x05Y\xfc8\x044\x00\x00\x00(\x04\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00aPK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00' + b'bPK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00' + b'\x00S\x00\x00\x00\x00\x00' + ) + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack() + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + zi = zh.remove('a') + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack([zi]) + + # local entry of 'a' should not be stripped (no valid entry) + fz = io.BytesIO(data) + with zipfile.ZipFile(fz, 'a') as zh: + zh.remove('a') + zh.repack() + + expected = ( + b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' + b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' + b'\x1f\x00\xe0\xffPK\x03\x04\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00b\xed\xc0\x81\x08\x00\x00\x00\xc00\xd6\xfbK\\' + b'd\x0b`PK\x01\x02\x14\x00\x14\x00\x00\x00\x08\x00\xa0l' + b'H\x05\xe2\x1e8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00$\x00\x00\x00' + b'bPK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00' + b'\x00S\x00\x00\x00\x00\x00' + ) + fz.seek(0) + self.assertEqual(fz.read(), expected) + + def test_partial_overlap_at_dd(self): + # file 'a' has an unsigned data descriptor (whose information isn't + # consistent with in central directory) that starts at the starting + # position of file 'b' + data = ( + b'PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00acontent' + b'PK\x03\x04\x14\x00\x00\x00\x00\x00\x00\x00!\x00\xa90\xc5\xfe' + b'\x07\x00\x00\x00\x07\x00\x00\x00\x01\x00\x00\x00bcontent' + b'PK\x01\x02\x14\x00\x14\x00\x08\x00\x00\x00\x00\x00!\x00' + b'\xa90\xc5\xfe\x07\x00\x00\x00\x07\x00\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01\x00\x00\x00\x00a' + b'PK\x01\x02\x14\x00\x14\x00\x00\x00\x00\x00\x00\x00!\x00' + b'\xa90\xc5\xfe\x07\x00\x00\x00\x07\x00\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01&\x00\x00\x00b' + b'PK\x05\x06\x00\x00\x00\x00\x02\x00\x02\x00^\x00\x00\x00L\x00' + b'\x00\x00\x00\x00' + ) + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + zi = zh.getinfo('a') + self.assertEqual(zi.header_offset, 0) + self.assertEqual(zi.compress_size, 7) + self.assertEqual(zi.file_size, 7) + self.assertEqual(zi.flag_bits, 8) + zi = zh.getinfo('b') + self.assertEqual(zi.header_offset, 38) + self.assertEqual(zi.compress_size, 7) + self.assertEqual(zi.file_size, 7) + self.assertEqual(zi.flag_bits, 0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack() + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + zi = zh.remove('a') + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack([zi]) + + # local entry of 'a' should not be stripped (no valid entry) + fz = io.BytesIO(data) + with zipfile.ZipFile(fz, 'a') as zh: + zh.remove('a') + zh.repack() + + expected = ( + b'PK\x03\x04\x14\x00\x08\x00\x00\x00\x00\x00!\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00acontent' + b'PK\x03\x04\x14\x00\x00\x00\x00\x00\x00\x00!\x00\xa90\xc5\xfe' + b'\x07\x00\x00\x00\x07\x00\x00\x00\x01\x00\x00\x00bcontent' + b'PK\x01\x02\x14\x00\x14\x00\x00\x00\x00\x00\x00\x00!\x00' + b'\xa90\xc5\xfe\x07\x00\x00\x00\x07\x00\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01&\x00\x00\x00b' + b'PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00L\x00' + b'\x00\x00\x00\x00' + ) + fz.seek(0) + self.assertEqual(fz.read(), expected) + + def test_overlap_with_central_dir(self): + # see `test_overlap_with_central_dir` in built-in test.test_zipfile + data = ( + b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' + b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\xb4\x81\x00\x00\x00\x00aP' + b'K\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00' + ) + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number for file header'): + zh.repack() + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + zi = zh.remove('a') + with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number for file header'): + zh.repack([zi]) + + # local entry of 'a' should not be stripped (not found) + fz = io.BytesIO(data) + with zipfile.ZipFile(fz, 'a') as zh: + zh.remove('a') + zh.repack() + + expected = ( + b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00' + ) + fz.seek(0) + self.assertEqual(fz.read(), expected) + + def test_overlap_with_archive_comment(self): + # see `test_overlap_with_archive_comment` in built-in test.test_zipfile + data = ( + b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' + b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x00\xb4\x81E\x00\x00\x00aP' + b'K\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00/\x00\x00\x00\x00' + b'\x00\x00\x00*\x00' + b'PK\x03\x04\x14\x00\x00\x00\x08\x00G_|Z\xe2\x1e' + b'8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00aK' + b'L\x1c\x05\xa3`\x14\x8cx\x00\x00' + ) + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack() + + with zipfile.ZipFile(io.BytesIO(data), 'a') as zh: + zi = zh.remove('a') + with self.assertRaisesRegex(zipfile.BadZipFile, 'Overlapped entries'): + zh.repack([zi]) + + # local entry of 'a' should not be stripped (not found) + fz = io.BytesIO(data) + with zipfile.ZipFile(fz, 'a') as zh: + zh.remove('a') + zh.repack() + + expected = ( + b'PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00*\x00' + b'PK\x03\x04\x14\x00\x00\x00\x08\x00G_|Z\xe2\x1e' + b'8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00aK' + b'L\x1c\x05\xa3`\x14\x8cx\x00\x00' + ) + fz.seek(0) + self.assertEqual(fz.read(), expected) + +class ZipRepackerTests(unittest.TestCase): + def _generate_local_file_entry(self, arcname, raw_bytes, + compression=zipfile.ZIP_STORED, + force_zip64=False, dd=False, dd_sig=True): + fz = io.BytesIO() + f = Unseekable(fz) if dd else fz + cm = (mock.patch.object(struct, 'pack', side_effect=struct_pack_no_dd_sig) + if dd and not dd_sig else contextlib.nullcontext()) + with zipfile.ZipFile(f, 'w', compression=compression) as zh: + with cm, zh.open(arcname, 'w', force_zip64=force_zip64) as fh: + fh.write(raw_bytes) + if dd: + zi = zh.infolist()[0] + self.assertTrue(zi.flag_bits & zipfile._MASK_USE_DATA_DESCRIPTOR, + f'data descriptor flag not set: {zi.filename}') + fz.seek(0) + return fz.read() + + def test_validate_local_file_entry_stored(self): + self._test_validate_local_file_entry(method=zipfile.ZIP_STORED) + + @requires_zlib() + def test_validate_local_file_entry_zlib(self): + self._test_validate_local_file_entry(method=zipfile.ZIP_DEFLATED) + + @requires_bz2() + def test_validate_local_file_entry_bz2(self): + self._test_validate_local_file_entry(method=zipfile.ZIP_BZIP2) + + @requires_lzma() + def test_validate_local_file_entry_lzma(self): + self._test_validate_local_file_entry(method=zipfile.ZIP_LZMA) + + @requires_zstd() + def test_validate_local_file_entry_zstd(self): + self._test_validate_local_file_entry(method=zipfile.ZIP_ZSTANDARD) + + def _test_validate_local_file_entry(self, method): + # strict_descriptor=False to exercise unsigned data descriptor scanning + # (the default is strict_descriptor=True, tested separately below) + repacker = zipfile._ZipRepacker(strict_descriptor=False) + + # basic + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, len(bytes_)) + m_sdd.assert_not_called() + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + # offset + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_) + 1) + self.assertEqual(result, len(bytes_)) + m_sdd.assert_not_called() + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + bytes_ = b'pre' + bytes_ + b'post' + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 3, len(bytes_) - 4) + self.assertEqual(result, len(bytes_) - 7) + m_sdd.assert_not_called() + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 3, len(bytes_)) + self.assertEqual(result, len(bytes_) - 7) + m_sdd.assert_not_called() + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + # return None if no match at given offset + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 2, len(bytes_) - 4) + self.assertEqual(result, None) + m_sdd.assert_not_called() + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 4, len(bytes_) - 4) + self.assertEqual(result, None) + m_sdd.assert_not_called() + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + # return None if truncated local file header + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method) + bytes_ = bytes_[:zipfile.sizeFileHeader - 1] + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, None) + m_sdd.assert_not_called() + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + # data descriptor + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method, dd=True) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, len(bytes_)) + m_sdd.assert_called_once_with(fz, 38, len(bytes_), False) + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + # data descriptor (unsigned) + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method, dd=True, dd_sig=False) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, len(bytes_)) + m_sdd.assert_called_once_with(fz, 38, len(bytes_), False) + m_sddnsbd.assert_called_once_with(fz, 38, len(bytes_), False, method) + if repacker._scan_data_descriptor_no_sig_by_decompression(fz, 38, len(bytes_), False, method): + m_sddns.assert_not_called() + else: + m_sddns.assert_called_once_with(fz, 38, len(bytes_), False) + + # return None for data descriptor (unsigned) if `strict_descriptor=True` + repacker = zipfile._ZipRepacker(strict_descriptor=True) + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method, dd=True, dd_sig=False) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, None) + m_sdd.assert_called_once_with(fz, 38, len(bytes_), False) + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + def test_validate_local_file_entry_zip64_stored(self): + self._test_validate_local_file_entry_zip64(method=zipfile.ZIP_STORED) + + @requires_zlib() + def test_validate_local_file_entry_zip64_zlib(self): + self._test_validate_local_file_entry_zip64(method=zipfile.ZIP_DEFLATED) + + @requires_bz2() + def test_validate_local_file_entry_zip64_bz2(self): + self._test_validate_local_file_entry_zip64(method=zipfile.ZIP_BZIP2) + + @requires_lzma() + def test_validate_local_file_entry_zip64_lzma(self): + self._test_validate_local_file_entry_zip64(method=zipfile.ZIP_LZMA) + + @requires_zstd() + def test_validate_local_file_entry_zip64_zstd(self): + self._test_validate_local_file_entry_zip64(method=zipfile.ZIP_ZSTANDARD) + + def _test_validate_local_file_entry_zip64(self, method): + # strict_descriptor=False to exercise unsigned data descriptor scanning + # (the default is strict_descriptor=True, tested separately below) + repacker = zipfile._ZipRepacker(strict_descriptor=False) + + # zip64 + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method, force_zip64=True) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, len(bytes_)) + m_sdd.assert_not_called() + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + # data descriptor + zip64 + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method, force_zip64=True, dd=True) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, len(bytes_)) + m_sdd.assert_called_once_with(fz, 58, len(bytes_), True) + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + # data descriptor (unsigned) + zip64 + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method, force_zip64=True, dd=True, dd_sig=False) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, len(bytes_)) + m_sdd.assert_called_once_with(fz, 58, len(bytes_), True) + m_sddnsbd.assert_called_once_with(fz, 58, len(bytes_), True, method) + if repacker._scan_data_descriptor_no_sig_by_decompression(fz, 58, len(bytes_), True, method): + m_sddns.assert_not_called() + else: + m_sddns.assert_called_once_with(fz, 58, len(bytes_), True) + + # return None for data descriptor (unsigned) if `strict_descriptor=True` + repacker = zipfile._ZipRepacker(strict_descriptor=True) + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', compression=method, force_zip64=True, dd=True, dd_sig=False) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, None) + m_sdd.assert_called_once_with(fz, 58, len(bytes_), True) + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + def test_validate_local_file_entry_overshoot(self): + """A header whose compress_size points past end_offset is rejected.""" + repacker = zipfile._ZipRepacker() + + bytes_ = self._generate_local_file_entry('file.txt', b'dummy') + fz = io.BytesIO(bytes_) + # exact fit at end_offset is accepted + self.assertEqual( + repacker._validate_local_file_entry(fz, 0, len(bytes_)), + len(bytes_), + ) + # one byte short: entry would extend past end_offset + self.assertIsNone( + repacker._validate_local_file_entry(fz, 0, len(bytes_) - 1), + ) + + # zip64 extra supplying the oversized compress_size + bytes_ = self._generate_local_file_entry( + 'file.txt', b'dummy', force_zip64=True) + fz = io.BytesIO(bytes_) + self.assertEqual( + repacker._validate_local_file_entry(fz, 0, len(bytes_)), + len(bytes_), + ) + self.assertIsNone( + repacker._validate_local_file_entry(fz, 0, len(bytes_) - 1), + ) + + def test_validate_local_file_entry_encrypted(self): + # strict_descriptor=False to exercise unsigned data descriptor scanning + # of an encrypted entry (the default strict_descriptor=True is tested below) + repacker = zipfile._ZipRepacker(strict_descriptor=False) + + bytes_ = ( + b'PK\x03\x04' + b'\x14\x00' + b'\x09\x00' + b'\x08\x00' + b'\xAB\x28' + b'\xD2\x5A' + b'\x00\x00\x00\x00' + b'\x00\x00\x00\x00' + b'\x00\x00\x00\x00' + b'\x08\x00' + b'\x00\x00' + b'file.txt' + b'\x97\xF1\x83\x34\x9D\xC4\x8C\xD3\xED\x79\x8C\xA2\xBB\x49\xFF\x1B\x89' + b'\x3F\xF2\xF4\x4F' + b'\x11\x00\x00\x00' + b'\x05\x00\x00\x00' + ) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, len(bytes_)) + m_sdd.assert_called_once_with(fz, 38, len(bytes_), False) + m_sddnsbd.assert_not_called() + m_sddns.assert_called_once_with(fz, 38, len(bytes_), False) + + # return None for the unsigned data descriptor if `strict_descriptor=True` + repacker = zipfile._ZipRepacker(strict_descriptor=True) + fz = io.BytesIO(bytes_) + with mock.patch.object(repacker, '_scan_data_descriptor', + wraps=repacker._scan_data_descriptor) as m_sdd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig_by_decompression', + wraps=repacker._scan_data_descriptor_no_sig_by_decompression) as m_sddnsbd, \ + mock.patch.object(repacker, '_scan_data_descriptor_no_sig', + wraps=repacker._scan_data_descriptor_no_sig) as m_sddns: + result = repacker._validate_local_file_entry(fz, 0, len(bytes_)) + self.assertEqual(result, None) + m_sdd.assert_called_once_with(fz, 38, len(bytes_), False) + m_sddnsbd.assert_not_called() + m_sddns.assert_not_called() + + def test_iter_scan_signature(self): + bytes_ = b'sig__sig__sig__sig' + ln = len(bytes_) + fp = io.BytesIO(bytes_) + repacker = zipfile._ZipRepacker() + + # basic + self.assertEqual( + list(repacker._iter_scan_signature(fp, b'sig', 0, ln)), + [0, 5, 10, 15], + ) + + # start_offset + self.assertEqual( + list(repacker._iter_scan_signature(fp, b'sig', 1, ln)), + [5, 10, 15], + ) + self.assertEqual( + list(repacker._iter_scan_signature(fp, b'sig', 6, ln)), + [10, 15], + ) + self.assertEqual( + list(repacker._iter_scan_signature(fp, b'sig', 16, ln)), + [], + ) + + # end_offset + self.assertEqual( + list(repacker._iter_scan_signature(fp, b'sig', 0, ln - 1)), + [0, 5, 10], + ) + self.assertEqual( + list(repacker._iter_scan_signature(fp, b'sig', 0, ln - 6)), + [0, 5], + ) + + # chunk_size + self.assertEqual( + list(repacker._iter_scan_signature(fp, b'sig', 0, ln, 3)), + [0, 5, 10, 15], + ) + self.assertEqual( + list(repacker._iter_scan_signature(fp, b'sig', 0, ln, 1)), + [0, 5, 10, 15], + ) + + def test_scan_data_descriptor(self): + repacker = zipfile._ZipRepacker() + + sig = zipfile._DD_SIGNATURE + raw_bytes = comp_bytes = b'dummy' + raw_len = comp_len = len(raw_bytes) + raw_crc = zipfile.crc32(raw_bytes) + + # basic + bytes_ = comp_bytes + struct.pack('<4L', sig, raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False), + (raw_crc, comp_len, raw_len, 16), + ) + + # return None if no signature + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + # return None if compressed size not match + bytes_ = comp_bytes + struct.pack('<4L', sig, raw_crc, comp_len + 1, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + bytes_ = comp_bytes + struct.pack('<4L', sig, raw_crc, comp_len - 1, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + bytes_ = b'1' + comp_bytes + struct.pack('<4L', sig, raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + bytes_ = comp_bytes[1:] + struct.pack('<4L', sig, raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + # zip64 + bytes_ = comp_bytes + struct.pack('<2L2Q', sig, raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), True), + (raw_crc, comp_len, raw_len, 24), + ) + + # offset + bytes_ = comp_bytes + struct.pack('<4L', sig, raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 1, len(bytes_), False), + None, + ) + + bytes_ = b'123' + comp_bytes + struct.pack('<4L', sig, raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 3, len(bytes_), False), + (raw_crc, comp_len, raw_len, 16), + ) + + # end_offset + bytes_ = comp_bytes + struct.pack('<4L', sig, raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_) - 1, False), + None, + ) + + bytes_ = comp_bytes + struct.pack('<4L', sig, raw_crc, comp_len, raw_len) + b'123' + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_) - 3, False), + (raw_crc, comp_len, raw_len, 16), + ) + self.assertEqual( + repacker._scan_data_descriptor(io.BytesIO(bytes_), 0, len(bytes_), False), + (raw_crc, comp_len, raw_len, 16), + ) + + def test_scan_data_descriptor_no_sig(self): + repacker = zipfile._ZipRepacker() + + raw_bytes = comp_bytes = b'dummy' + raw_len = comp_len = len(raw_bytes) + raw_crc = zipfile.crc32(raw_bytes) + + # basic + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False), + (raw_crc, comp_len, raw_len, 12), + ) + + # return None if compressed size not match + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len + 1, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len - 1, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + bytes_ = b'1' + comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + bytes_ = comp_bytes[1:] + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False), + None, + ) + + # zip64 + bytes_ = comp_bytes + struct.pack('<L2Q', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), True), + (raw_crc, comp_len, raw_len, 20), + ) + + # offset + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 1, len(bytes_), False), + None, + ) + + bytes_ = b'123' + comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 3, len(bytes_), False), + (raw_crc, comp_len, raw_len, 12), + ) + + # end_offset + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_) - 1, False), + None, + ) + + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + b'123' + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_) - 3, False), + (raw_crc, comp_len, raw_len, 12), + ) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False), + (raw_crc, comp_len, raw_len, 12), + ) + + # chunk_size + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False, 12), + (raw_crc, comp_len, raw_len, 12), + ) + self.assertEqual( + repacker._scan_data_descriptor_no_sig(io.BytesIO(bytes_), 0, len(bytes_), False, 1), + (raw_crc, comp_len, raw_len, 12), + ) + + def test_scan_data_descriptor_no_sig_by_decompression_stored(self): + self._test_scan_data_descriptor_no_sig_by_decompression_invalid(zipfile.ZIP_STORED) + + @requires_zlib() + def test_scan_data_descriptor_no_sig_by_decompression_zlib(self): + self._test_scan_data_descriptor_no_sig_by_decompression(zipfile.ZIP_DEFLATED) + + @requires_bz2() + def test_scan_data_descriptor_no_sig_by_decompression_bz2(self): + self._test_scan_data_descriptor_no_sig_by_decompression(zipfile.ZIP_BZIP2) + + @requires_lzma() + def test_scan_data_descriptor_no_sig_by_decompression_lzma(self): + self._test_scan_data_descriptor_no_sig_by_decompression(zipfile.ZIP_LZMA) + + @requires_zstd() + def test_scan_data_descriptor_no_sig_by_decompression_zstd(self): + self._test_scan_data_descriptor_no_sig_by_decompression(zipfile.ZIP_ZSTANDARD) + + def test_scan_data_descriptor_no_sig_by_decompression_unknown(self): + method = 1024 # simulate an unknown method + self._test_scan_data_descriptor_no_sig_by_decompression_invalid(method) + + def _test_scan_data_descriptor_no_sig_by_decompression(self, method): + repacker = zipfile._ZipRepacker() + + raw_bytes = b'dummy' + raw_len = len(raw_bytes) + raw_crc = zipfile.crc32(raw_bytes) + + compressor = zipfile._get_compressor(method) + comp_bytes = compressor.compress(raw_bytes) + comp_bytes += compressor.flush() + comp_len = len(comp_bytes) + + # basic + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, len(bytes_), False, method), + (raw_crc, comp_len, raw_len, 12), + ) + + # return None if data length < DD signature + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, 11, False, method), + None, + ) + + # return None if compressed size not match + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len + 1, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, len(bytes_), False, method), + None, + ) + + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len - 1, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, len(bytes_), False, method), + None, + ) + + # zip64 + bytes_ = comp_bytes + struct.pack('<L2Q', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, len(bytes_), True, method), + (raw_crc, comp_len, raw_len, 20), + ) + + # offset + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 1, len(bytes_), False, method), + None, + ) + + bytes_ = b'123' + comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 3, len(bytes_), False, method), + (raw_crc, comp_len, raw_len, 12), + ) + + # end_offset + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, len(bytes_) - 1, False, method), + None, + ) + + bytes_ = comp_bytes + struct.pack('<3L', raw_crc, comp_len, raw_len) + b'123' + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, len(bytes_) - 3, False, method), + (raw_crc, comp_len, raw_len, 12), + ) + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, len(bytes_), False, method), + (raw_crc, comp_len, raw_len, 12), + ) + + def _test_scan_data_descriptor_no_sig_by_decompression_invalid(self, method): + repacker = zipfile._ZipRepacker() + + bytes_ = b'dummy' + self.assertEqual( + repacker._scan_data_descriptor_no_sig_by_decompression( + io.BytesIO(bytes_), 0, len(bytes_), False, method), + False, + ) + + @requires_zlib() + def test_trace_compressed_block_end_zlib(self): + import zlib + self._test_trace_compressed_block_end(zipfile.ZIP_DEFLATED, zlib.error) + + @requires_bz2() + def test_trace_compressed_block_end_bz2(self): + self._test_trace_compressed_block_end(zipfile.ZIP_BZIP2, OSError) + + @requires_lzma() + def test_trace_compressed_block_end_lzma(self): + self._test_trace_compressed_block_end(zipfile.ZIP_LZMA, EOFError) + + @requires_zstd() + def test_trace_compressed_block_end_zstd(self): + import compression.zstd + self._test_trace_compressed_block_end(zipfile.ZIP_ZSTANDARD, compression.zstd.ZstdError) + + def _test_trace_compressed_block_end(self, method, exc_cls): + repacker = zipfile._ZipRepacker() + + compressor = zipfile._get_compressor(method) + + comp_bytes = compressor.compress(b'dummy') + comp_bytes += compressor.flush() + comp_len = len(comp_bytes) + + # basic + decompressor = zipfile._get_decompressor(method) + bytes_ = comp_bytes + self.assertEqual( + repacker._trace_compressed_block_end(io.BytesIO(bytes_), 0, len(bytes_), decompressor), + comp_len, + ) + + # offset + decompressor = zipfile._get_decompressor(method) + bytes_ = comp_bytes + with self.assertRaises(exc_cls): + repacker._trace_compressed_block_end(io.BytesIO(bytes_), 1, len(bytes_), decompressor) + + decompressor = zipfile._get_decompressor(method) + bytes_ = b'123' + comp_bytes + with self.assertRaises(exc_cls): + repacker._trace_compressed_block_end(io.BytesIO(bytes_), 0, len(bytes_), decompressor) + + decompressor = zipfile._get_decompressor(method) + bytes_ = b'123' + comp_bytes + self.assertEqual( + repacker._trace_compressed_block_end(io.BytesIO(bytes_), 3, len(bytes_), decompressor), + comp_len + 3, + ) + + # end_offset + decompressor = zipfile._get_decompressor(method) + bytes_ = comp_bytes + with self.assertRaises(EOFError): + repacker._trace_compressed_block_end(io.BytesIO(bytes_), 0, len(bytes_) - 1, decompressor) + + decompressor = zipfile._get_decompressor(method) + bytes_ = comp_bytes + b'123' + self.assertEqual( + repacker._trace_compressed_block_end(io.BytesIO(bytes_), 0, len(bytes_) - 3, decompressor), + comp_len, + ) + + # chunk_size + decompressor = zipfile._get_decompressor(method) + bytes_ = comp_bytes + self.assertEqual( + repacker._trace_compressed_block_end(io.BytesIO(bytes_), 0, len(bytes_), decompressor, 16), + comp_len, + ) + + decompressor = zipfile._get_decompressor(method) + bytes_ = comp_bytes + self.assertEqual( + repacker._trace_compressed_block_end(io.BytesIO(bytes_), 0, len(bytes_), decompressor, 1), + comp_len, + ) + + def test_calc_local_file_entry_size(self): + repacker = zipfile._ZipRepacker() + + # basic + fz = io.BytesIO() + with zipfile.ZipFile(fz, 'w') as zh: + with zh.open('file.txt', 'w') as fh: + fh.write(b'dummy') + zi = zh.infolist()[-1] + + self.assertEqual( + repacker._calc_local_file_entry_size(fz, zi), + 43, + ) + + # data descriptor + fz = io.BytesIO() + with zipfile.ZipFile(Unseekable(fz), 'w') as zh: + with zh.open('file.txt', 'w') as fh: + fh.write(b'dummy') + zi = zh.infolist()[-1] + + self.assertEqual( + repacker._calc_local_file_entry_size(fz, zi), + 59, + ) + + # data descriptor (unsigned) + fz = io.BytesIO() + with zipfile.ZipFile(Unseekable(fz), 'w') as zh: + with mock.patch.object(struct, 'pack', side_effect=struct_pack_no_dd_sig), \ + zh.open('file.txt', 'w') as fh: + fh.write(b'dummy') + zi = zh.infolist()[-1] + + self.assertEqual( + repacker._calc_local_file_entry_size(fz, zi), + 55, + ) + + def test_calc_local_file_entry_size_zip64(self): + repacker = zipfile._ZipRepacker() + + # zip64 + fz = io.BytesIO() + with zipfile.ZipFile(fz, 'w') as zh: + with zh.open('file.txt', 'w', force_zip64=True) as fh: + fh.write(b'dummy') + zi = zh.infolist()[-1] + + self.assertEqual( + repacker._calc_local_file_entry_size(fz, zi), + 63, + ) + + # data descriptor + zip64 + fz = io.BytesIO() + with zipfile.ZipFile(Unseekable(fz), 'w') as zh: + with zh.open('file.txt', 'w', force_zip64=True) as fh: + fh.write(b'dummy') + zi = zh.infolist()[-1] + + self.assertEqual( + repacker._calc_local_file_entry_size(fz, zi), + 87, + ) + + # data descriptor (unsigned) + zip64 + fz = io.BytesIO() + with zipfile.ZipFile(Unseekable(fz), 'w') as zh: + with mock.patch.object(struct, 'pack', side_effect=struct_pack_no_dd_sig), \ + zh.open('file.txt', 'w', force_zip64=True) as fh: + fh.write(b'dummy') + zi = zh.infolist()[-1] + + self.assertEqual( + repacker._calc_local_file_entry_size(fz, zi), + 83, + ) + + def test_copy_bytes(self): + repacker = zipfile._ZipRepacker() + + fp = io.BytesIO(b'abc123') + repacker._copy_bytes(fp, 0, 3, 3) + self.assertEqual(fp.getvalue(), b'abcabc') + + fp = io.BytesIO(b'abc123') + repacker._copy_bytes(fp, 3, 0, 3) + self.assertEqual(fp.getvalue(), b'123123') + + fp = io.BytesIO(b'abc123') + repacker._copy_bytes(fp, 0, 4, 3) + self.assertEqual(fp.getvalue(), b'abc1abc') + + fp = io.BytesIO(b'abc123') + repacker._copy_bytes(fp, 0, 10, 3) + self.assertEqual(fp.getvalue(), b'abc123\x00\x00\x00\x00abc') + + fp = io.BytesIO(b'abc123') + repacker._copy_bytes(fp, 0, 3, 2) + self.assertEqual(fp.getvalue(), b'abcab3') + + # check chunk_size + repacker = zipfile._ZipRepacker(chunk_size=6) + fp = io.BytesIO(b'abcdef123456') + with mock.patch.object(fp, 'read', wraps=fp.read) as m_read: + repacker._copy_bytes(fp, 0, 12, 6) + self.assertEqual(fp.getvalue(), b'abcdef123456abcdef') + m_read.assert_called_once_with(6) + + repacker = zipfile._ZipRepacker(chunk_size=3) + fp = io.BytesIO(b'abcdef123456') + with mock.patch.object(fp, 'read', wraps=fp.read) as m_read: + repacker._copy_bytes(fp, 0, 12, 6) + self.assertEqual(fp.getvalue(), b'abcdef123456abcdef') + self.assertEqual(m_read.mock_calls, [mock.call(3), mock.call(3)]) + + repacker = zipfile._ZipRepacker(chunk_size=1) + fp = io.BytesIO(b'abcdef123456') + with mock.patch.object(fp, 'read', wraps=fp.read) as m_read: + repacker._copy_bytes(fp, 0, 12, 6) + self.assertEqual(fp.getvalue(), b'abcdef123456abcdef') + self.assertEqual(m_read.mock_calls, [ + mock.call(1), mock.call(1), mock.call(1), mock.call(1), mock.call(1), mock.call(1)]) + + def test_copy_bytes_short_read(self): + """Raise rather than loop forever if EOF is hit before size bytes.""" + repacker = zipfile._ZipRepacker() + fp = io.BytesIO(b'abc123') + with self.assertRaises(zipfile.BadZipFile): + repacker._copy_bytes(fp, 0, 0, 100) + + class PyZipFileTests(unittest.TestCase): def assertCompiledIn(self, name, namelist): if name + 'o' not in namelist: @@ -1886,11 +4006,8 @@ def test_write_with_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_source_date_epoch.txt") - get_time = time.localtime(int(os.environ['SOURCE_DATE_EPOCH']))[:6] - # Compare each element of the date_time tuple - # Allow for a 1-second difference - for z_time, g_time in zip(zip_info.date_time, get_time): - self.assertAlmostEqual(z_time, g_time, delta=1) + expected_utc = (2025, 1, 1, 7, 19, 58) + self.assertEqual(zip_info.date_time, expected_utc) def test_write_without_source_date_epoch(self): with os_helper.EnvironmentVarGuard() as env: @@ -1901,9 +4018,13 @@ def test_write_without_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_no_source_date_epoch.txt") - current_time = time.localtime()[:6] - for z_time, c_time in zip(zip_info.date_time, current_time): - self.assertAlmostEqual(z_time, c_time, delta=1) + self.assertTimestampAlmostEqual(time.localtime(), zip_info.date_time, tolerance=2) + + def assertTimestampAlmostEqual(self, time1, time2, tolerance): + import datetime + dt1 = datetime.datetime(*time1[:6]) + dt2 = datetime.datetime(*time2[:6]) + self.assertLessEqual((dt1 - dt2).total_seconds(), tolerance) def test_close(self): """Check that the zipfile is closed after the 'with' block.""" @@ -1968,8 +4089,7 @@ def create_zipfile_with_extra_data(self, filename, extra_data_name): tag_for_unicode_path = b'\x75\x70' version_of_unicode_path = b'\x01' - import zlib - filename_crc = struct.pack('<L', zlib.crc32(filename_encoded)) + filename_crc = struct.pack('<L', zipfile.crc32(filename_encoded)) extra_data = version_of_unicode_path + filename_crc + extra_data_name tsize = len(extra_data).to_bytes(2, 'little') @@ -1979,19 +4099,16 @@ def create_zipfile_with_extra_data(self, filename, extra_data_name): # add the file to the ZIP archive zf.writestr(zip_info, b'Hello World!') - @requires_zlib() def test_read_zipfile_containing_unicode_path_extra_field(self): self.create_zipfile_with_extra_data("이름.txt", "이름.txt".encode("utf-8")) with zipfile.ZipFile(TESTFN, "r") as zf: self.assertEqual(zf.filelist[0].filename, "이름.txt") - @requires_zlib() def test_read_zipfile_warning(self): self.create_zipfile_with_extra_data("이름.txt", b"") with self.assertWarns(UserWarning): zipfile.ZipFile(TESTFN, "r").close() - @requires_zlib() def test_read_zipfile_error(self): self.create_zipfile_with_extra_data("이름.txt", b"\xff") with self.assertRaises(zipfile.BadZipfile): @@ -3639,29 +5756,23 @@ def test_read_with_unsuitable_metadata_encoding(self): def test_read_after_append(self): newname = '\u56db' # Han 'four' - expected_names = [name.encode('shift_jis').decode('cp437') - for name in self.file_names[:2]] + self.file_names[2:] - expected_names.append(newname) - expected_content = (*self.file_content, b"newcontent") + newname2 = 'fünf' # representable in cp437, but still stored as UTF-8 + expected_names = [*self.file_names, newname, newname2] + mojibake_expected_names = [name.encode('shift_jis').decode('cp437') + if i < 2 else name + for i, name in enumerate(expected_names)] + expected_content = (*self.file_content, b"newcontent", b"newcontent2") with zipfile.ZipFile(TESTFN, "a") as zipfp: zipfp.writestr(newname, "newcontent") - self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) + zipfp.writestr(newname2, "newcontent2") + self.assertEqual(sorted(zipfp.namelist()), sorted(mojibake_expected_names)) with zipfile.ZipFile(TESTFN, "r") as zipfp: - self._test_read(zipfp, expected_names, expected_content) + self._test_read(zipfp, mojibake_expected_names, expected_content) with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: - self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) - for i, (name, content) in enumerate(zip(expected_names, expected_content)): - info = zipfp.getinfo(name) - self.assertEqual(info.filename, name) - self.assertEqual(info.file_size, len(content)) - if i < 2: - with self.assertRaises(zipfile.BadZipFile): - zipfp.read(name) - else: - self.assertEqual(zipfp.read(name), content) + self._test_read(zipfp, expected_names, expected_content) def test_write_with_metadata_encoding(self): ZF = zipfile.ZipFile @@ -3670,6 +5781,20 @@ def test_write_with_metadata_encoding(self): "^metadata_encoding is only"): ZF("nonesuch.zip", mode, metadata_encoding="shift_jis") + def test_add_comment(self): + with zipfile.ZipFile(TESTFN, "r") as zipfp: + mojibake_expected_names = zipfp.namelist() + + with zipfile.ZipFile(TESTFN, "a") as zipfp: + zipfp.comment = b'comment' + self.assertEqual(zipfp.namelist(), mojibake_expected_names) + + with zipfile.ZipFile(TESTFN, "r") as zipfp: + self._test_read(zipfp, mojibake_expected_names, self.file_content) + + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: + self._test_read(zipfp, self.file_names, self.file_content) + def test_cli_with_metadata_encoding(self): errmsg = "Non-conforming encodings not supported with -c." args = ["--metadata-encoding=shift_jis", "-c", "nonesuch", "nonesuch"] diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index 2e1affe02528581..7d802d59849ce1f 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -13,12 +13,16 @@ import zipfile, unittest import time +import tracemalloc import sys +import unittest.mock as mock from tempfile import TemporaryFile from test.support import os_helper from test.support import requires_zlib +from test.test_zipfile.test_core import Unseekable +from test.test_zipfile.test_core import struct_pack_no_dd_sig TESTFN = os_helper.TESTFN TESTFN2 = TESTFN + "2" @@ -87,6 +91,178 @@ def tearDown(self): os_helper.unlink(TESTFN2) +class TestRepack(unittest.TestCase): + def setUp(self): + # Create test data. + line_gen = ("Test of zipfile line %d." % i for i in range(1000000)) + self.data = '\n'.join(line_gen).encode('ascii') + + # It will contain enough copies of self.data to reach about 8 GiB. + self.datacount = 8*1024**3 // len(self.data) + + # memory usage should not exceed 10 MiB + self.allowed_memory = 10*1024**2 + + def _write_large_file(self, fh): + next_time = time.monotonic() + _PRINT_WORKING_MSG_INTERVAL + for num in range(self.datacount): + fh.write(self.data) + # Print still working message since this test can be really slow + if next_time <= time.monotonic(): + next_time = time.monotonic() + _PRINT_WORKING_MSG_INTERVAL + print(( + ' writing %d of %d, be patient...' % + (num, self.datacount)), file=sys.__stdout__) + sys.__stdout__.flush() + + def test_strip_removed_large_file(self): + """Should move the physical data of a file positioned after a large + removed file without causing a memory issue.""" + # Try the temp file. If we do TESTFN2, then it hogs + # gigabytes of disk space for the duration of the test. + with TemporaryFile() as f: + tracemalloc.start() + self._test_strip_removed_large_file(f) + self.assertFalse(f.closed) + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + self.assertLess(peak, self.allowed_memory) + + def _test_strip_removed_large_file(self, f): + file = 'file.txt' + file1 = 'largefile.txt' + data = b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem' + with zipfile.ZipFile(f, 'w') as zh: + with zh.open(file1, 'w', force_zip64=True) as fh: + self._write_large_file(fh) + zh.writestr(file, data) + + with zipfile.ZipFile(f, 'a') as zh: + zh.remove(file1) + zh.repack() + self.assertIsNone(zh.testzip()) + + def test_strip_removed_file_before_large_file(self): + """Should move the physical data of a large file positioned after a + removed file without causing a memory issue.""" + # Try the temp file. If we do TESTFN2, then it hogs + # gigabytes of disk space for the duration of the test. + with TemporaryFile() as f: + tracemalloc.start() + self._test_strip_removed_file_before_large_file(f) + self.assertFalse(f.closed) + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + self.assertLess(peak, self.allowed_memory) + + def _test_strip_removed_file_before_large_file(self, f): + file = 'file.txt' + file1 = 'largefile.txt' + data = b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem' + with zipfile.ZipFile(f, 'w') as zh: + zh.writestr(file, data) + with zh.open(file1, 'w', force_zip64=True) as fh: + self._write_large_file(fh) + + with zipfile.ZipFile(f, 'a') as zh: + zh.remove(file) + zh.repack() + self.assertIsNone(zh.testzip()) + + def test_strip_removed_large_file_with_dd(self): + """Should scan for the data descriptor of a removed large file without + causing a memory issue.""" + # Try the temp file. If we do TESTFN2, then it hogs + # gigabytes of disk space for the duration of the test. + with TemporaryFile() as f: + tracemalloc.start() + self._test_strip_removed_large_file_with_dd(f) + self.assertFalse(f.closed) + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + self.assertLess(peak, self.allowed_memory) + + def _test_strip_removed_large_file_with_dd(self, f): + file = 'file.txt' + file1 = 'largefile.txt' + data = b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem' + with zipfile.ZipFile(Unseekable(f), 'w') as zh: + with zh.open(file1, 'w', force_zip64=True) as fh: + self._write_large_file(fh) + zh.writestr(file, data) + + with zipfile.ZipFile(f, 'a') as zh: + zh.remove(file1) + zh.repack() + self.assertIsNone(zh.testzip()) + + def test_strip_removed_large_file_with_dd_no_sig(self): + """Should scan for the data descriptor (without signature) of a removed + large file without causing a memory issue.""" + # Reduce data scale for this test, as it's especially slow... + self.datacount = 30*1024**2 // len(self.data) + self.allowed_memory = 200*1024 + + # Try the temp file. If we do TESTFN2, then it hogs + # gigabytes of disk space for the duration of the test. + with TemporaryFile() as f: + tracemalloc.start() + self._test_strip_removed_large_file_with_dd_no_sig(f) + self.assertFalse(f.closed) + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + self.assertLess(peak, self.allowed_memory) + + def _test_strip_removed_large_file_with_dd_no_sig(self, f): + file = 'file.txt' + file1 = 'largefile.txt' + data = b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem' + with mock.patch('zipfile.struct.pack', side_effect=struct_pack_no_dd_sig): + with zipfile.ZipFile(Unseekable(f), 'w') as zh: + with zh.open(file1, 'w', force_zip64=True) as fh: + self._write_large_file(fh) + zh.writestr(file, data) + + with zipfile.ZipFile(f, 'a') as zh: + zh.remove(file1) + # strict_descriptor=False to scan the unsigned data descriptor + # (scanning is disabled under the strict_descriptor=True default) + zh.repack(strict_descriptor=False) + self.assertIsNone(zh.testzip()) + + @requires_zlib() + def test_strip_removed_large_file_with_dd_no_sig_by_decompression(self): + """Should scan for the data descriptor (without signature) of a removed + large file without causing a memory issue.""" + # Try the temp file. If we do TESTFN2, then it hogs + # gigabytes of disk space for the duration of the test. + with TemporaryFile() as f: + tracemalloc.start() + self._test_strip_removed_large_file_with_dd_no_sig_by_decompression( + f, zipfile.ZIP_DEFLATED) + self.assertFalse(f.closed) + current, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + self.assertLess(peak, self.allowed_memory) + + def _test_strip_removed_large_file_with_dd_no_sig_by_decompression(self, f, method): + file = 'file.txt' + file1 = 'largefile.txt' + data = b'Sed ut perspiciatis unde omnis iste natus error sit voluptatem' + with mock.patch('zipfile.struct.pack', side_effect=struct_pack_no_dd_sig): + with zipfile.ZipFile(Unseekable(f), 'w', compression=method) as zh: + with zh.open(file1, 'w', force_zip64=True) as fh: + self._write_large_file(fh) + zh.writestr(file, data) + + with zipfile.ZipFile(f, 'a') as zh: + zh.remove(file1) + # strict_descriptor=False to detect the unsigned data descriptor + # (scanning is disabled under the strict_descriptor=True default) + zh.repack(strict_descriptor=False) + self.assertIsNone(zh.testzip()) + + class OtherTests(unittest.TestCase): def testMoreThan64kFiles(self): # This test checks that more than 64k files can be added to an archive, diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 41366fbf443a4fc..2f213e34c2c329f 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -72,7 +72,7 @@ class TextWrapper: # Hello/ /there/ /--/ /you/ /goof-/ball,/ /use/ /the/ /-b/ /option! # (after stripping out empty strings). word_punct = r'[\w!"\'&.,?]' - letter = r'[^\d\W]' + letter = r'[\w--\d]' whitespace = r'[%s]' % re.escape(_whitespace) nowhitespace = '[^' + whitespace[1:] wordsep_re = re.compile(r''' diff --git a/Lib/timeit.py b/Lib/timeit.py index a897d9663c24e2e..01bdfd901e30d5e 100644 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -299,14 +299,14 @@ def main(args=None, *, _wrap_timer=None): "-p", "--process", action="store_true", - help="use time.process_time() (default is time.perf_counter())", + help="use `time.process_time()` (default is `time.perf_counter()`)", ) parser.add_argument( "-t", "--target-time", type=float, default=default_target_time, - help="if --number is 0 the code will run until it takes " + help="if `--number` is 0 the code will run until it takes " "at least this many seconds (default %(default)s)", ) parser.add_argument( diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index ba8365f56c37a70..2e1230dc449f96e 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -505,12 +505,14 @@ def trace_variable(self, mode, callback): Return the name of the callback. This deprecated method wraps a deprecated Tcl method removed - in Tcl 9.0. Use trace_add() instead. + in Tcl 9.0 and will be removed in Python 3.17. Use trace_add() + instead. """ import warnings warnings.warn( - "trace_variable() is deprecated and not supported with Tcl 9; " - "use trace_add() instead.", + "trace_variable() is deprecated and will be removed in Python " + "3.17; use trace_add() instead. It is not supported with " + "Tcl 9.", DeprecationWarning, stacklevel=2) cbname = self._register(callback) self._tk.call("trace", "variable", self._name, mode, cbname) @@ -525,12 +527,14 @@ def trace_vdelete(self, mode, cbname): CBNAME is the name of the callback returned from trace_variable or trace. This deprecated method wraps a deprecated Tcl method removed - in Tcl 9.0. Use trace_remove() instead. + in Tcl 9.0 and will be removed in Python 3.17. Use trace_remove() + instead. """ import warnings warnings.warn( - "trace_vdelete() is deprecated and not supported with Tcl 9; " - "use trace_remove() instead.", + "trace_vdelete() is deprecated and will be removed in Python " + "3.17; use trace_remove() instead. It is not supported with " + "Tcl 9.", DeprecationWarning, stacklevel=2) self._tk.call("trace", "vdelete", self._name, mode, cbname) cbname = self._tk.splitlist(cbname)[0] @@ -548,12 +552,14 @@ def trace_vinfo(self): """Return all trace callback information. This deprecated method wraps a deprecated Tcl method removed - in Tcl 9.0. Use trace_info() instead. + in Tcl 9.0 and will be removed in Python 3.17. Use trace_info() + instead. """ import warnings warnings.warn( - "trace_vinfo() is deprecated and not supported with Tcl 9; " - "use trace_info() instead.", + "trace_vinfo() is deprecated and will be removed in Python " + "3.17; use trace_info() instead. It is not supported with " + "Tcl 9.", DeprecationWarning, stacklevel=2) return [self._tk.splitlist(x) for x in self._tk.splitlist( self._tk.call("trace", "vinfo", self._name))] @@ -745,6 +751,73 @@ def tk_setPalette(self, *args, **kw): self.tk.call(('tk_setPalette',) + _flatten(args) + _flatten(list(kw.items()))) + def tk_appname(self, name=None): + """Query or set the name used to communicate with this application + through the send command. + + With no argument, return the current name; otherwise change it to NAME + and return the actual name set (which may have a suffix appended to + keep it unique).""" + if name is None: + return self.tk.call('tk', 'appname') + return self.tk.call('tk', 'appname', name) + + def tk_useinputmethods(self, boolean=None, *, displayof=0): + """Query or set whether Tk uses the X Input Methods (XIM) for filtering + events, and return the resulting state. + + This is significant only on X11; if XIM support is not available it + always returns False.""" + args = ('tk', 'useinputmethods') + self._displayof(displayof) + if boolean is not None: + args += (boolean,) + return self.tk.getboolean(self.tk.call(args)) + + def tk_caret(self, *, x=None, y=None, height=None): + """Set or query the caret location for this widget's display. + + The caret is the per-display insertion position used for global focus + indication (for accessibility) and for placing the input method + (XIM or IME) window. With no argument, return the current location as + a dictionary with keys 'x', 'y' and 'height'; otherwise update the + given coordinates.""" + args = ('tk', 'caret', self._w) + for option, value in (('-x', x), ('-y', y), ('-height', height)): + if value is not None: + args += (option, value) + if len(args) > 3: + self.tk.call(args) + else: + values = self.tk.splitlist(self.tk.call(args)) + return {values[i][1:]: self.tk.getint(values[i + 1]) + for i in range(0, len(values), 2)} + + def tk_scaling(self, number=None, *, displayof=0): + """Query or set the scaling factor used by Tk to convert between + physical units and pixels. + + The scaling factor is the number of pixels per point on the display, + where a point is 1/72 inch. With no argument, return the current + factor; otherwise set it to the floating-point NUMBER.""" + args = ('tk', 'scaling') + self._displayof(displayof) + if number is not None: + self.tk.call(args + (number,)) + else: + return self.tk.getdouble(self.tk.call(args)) + + def tk_inactive(self, reset=False, *, displayof=0): + """Return the number of milliseconds since the last time the user + interacted with the system, or -1 if the windowing system does not + support this. + + If RESET is true, reset the inactivity timer to zero instead and + return None.""" + args = ('tk', 'inactive') + self._displayof(displayof) + if reset: + self.tk.call(args + ('reset',)) + else: + return self.tk.getint(self.tk.call(args)) + def wait_variable(self, name='PY_VAR'): """Wait until the variable is modified. @@ -1275,6 +1348,15 @@ def winfo_interps(self, displayof=0): args = ('winfo', 'interps') + self._displayof(displayof) return self.tk.splitlist(self.tk.call(args)) + def winfo_isdark(self): # new in Tk 9.1 + """Return whether this widget is in "dark mode". + + On macOS and Windows return True if the widget is in "dark mode", + and False otherwise. Always return False on X11. + """ + return self.tk.getboolean( + self.tk.call('winfo', 'isdark', self._w)) + def winfo_ismapped(self): """Return true if this widget is mapped.""" return self.tk.getint( @@ -1503,13 +1585,26 @@ def bindtags(self, tagList=None): else: self.tk.call('bindtags', self._w, tagList) - def _bind(self, what, sequence, func, add, needcleanup=1): + def _delete_bind_commands(self, *what): + lines = self.tk.call(what).split('\n') + p = re.compile(r'if \{"\[([^ ]+) .*\]" == "break"\} break') + for line in lines: + m = p.fullmatch(line) + if m: + funcid = m[1] + try: + self.deletecommand(funcid) + except TclError: + pass + + def _bind(self, what, sequence, func, add): """Internal function.""" if isinstance(func, str): self.tk.call(what + (sequence, func)) elif func: - funcid = self._register(func, self._substitute, - needcleanup) + if not add: + self._delete_bind_commands(*what, sequence) + funcid = self._register(func, self._substitute, needcleanup=True) cmd = ('%sif {"[%s %s]" == "break"} break\n' % (add and '+' or '', @@ -1575,6 +1670,7 @@ def unbind(self, sequence, funcid=None): def _unbind(self, what, funcid=None): if funcid is None: + self._delete_bind_commands(*what) self.tk.call(*what, '') else: lines = self.tk.call(what).split('\n') @@ -1591,7 +1687,7 @@ def bind_all(self, sequence=None, func=None, add=None): An additional boolean parameter ADD specifies whether FUNC will be called additionally to the other bound function or whether it will replace the previous function. See bind for the return value.""" - return self._root()._bind(('bind', 'all'), sequence, func, add, True) + return self._root()._bind(('bind', 'all'), sequence, func, add) def unbind_all(self, sequence): """Unbind for all widgets for event SEQUENCE all functions.""" @@ -1605,7 +1701,7 @@ def bind_class(self, className, sequence=None, func=None, add=None): whether it will replace the previous function. See bind for the return value.""" - return self._root()._bind(('bind', className), sequence, func, add, True) + return self._root()._bind(('bind', className), sequence, func, add) def unbind_class(self, className, sequence): """Unbind for all widgets with bindtag CLASSNAME for event SEQUENCE @@ -2330,6 +2426,22 @@ def wm_group(self, pathName=None): group = wm_group + def wm_iconbadge(self, badge): # new in Tk 9.0 + """Set a badge for the icon of this widget. + + BADGE can be a positive integer number, for instance the number of + new or unread messages, or an exclamation point denoting attention + needed. If BADGE is an empty string, the badge image is removed from + the application icon. + + The badge is intended for display in the Dock (macOS), taskbar + (Windows) or app panel (X11). On X11, the variable + ::tk::icons::base_icon(WINDOW) must be set to the image used for the + window icon for this command to work.""" + return self.tk.call('wm', 'iconbadge', self._w, badge) + + iconbadge = wm_iconbadge + def wm_iconbitmap(self, bitmap=None, default=None): """Set bitmap for the iconified widget to BITMAP. Return the bitmap if None is given. @@ -2479,6 +2591,25 @@ def wm_sizefrom(self, who=None): sizefrom = wm_sizefrom + def wm_stackorder(self, relation=None, window=None): + """Query the stacking order of toplevel windows. + + If called with no arguments, return a list of toplevel widgets in + stacking order, from lowest to highest. The list recursively + includes all of this window's children that are toplevels; only + toplevels currently mapped to the screen are included. + + If RELATION is "isabove" or "isbelow" and WINDOW is another toplevel, + return True if this window is respectively above or below WINDOW in + the stacking order, and False otherwise.""" + if relation is None: + return [self._nametowidget(x) for x in self.tk.splitlist( + self.tk.call('wm', 'stackorder', self._w))] + return self.tk.getboolean( + self.tk.call('wm', 'stackorder', self._w, relation, window)) + + stackorder = wm_stackorder + def wm_state(self, newstate=None): """Query or set the state of this widget as one of normal, icon, iconic (see wm_iconwindow), withdrawn, or zoomed (Windows only).""" @@ -3011,7 +3142,7 @@ def addtag_withtag(self, newtag, tagOrId): """Add tag NEWTAG to all items with TAGORID.""" self.addtag(newtag, 'withtag', tagOrId) - def bbox(self, *args): + def bbox(self, *args): # overrides Misc.bbox """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle which encloses all items with tags specified as arguments.""" return self._getints( @@ -3150,7 +3281,7 @@ def find_withtag(self, tagOrId): """Return all items with TAGORID.""" return self.find('withtag', tagOrId) - def focus(self, *args): + def focus(self, *args): # overrides Misc.focus """Set focus to the first item specified in ARGS.""" return self.tk.call((self._w, 'focus') + args) @@ -3196,7 +3327,7 @@ def tag_lower(self, *args): (optional below another item).""" self.tk.call((self._w, 'lower') + args) - lower = tag_lower + lower = tag_lower # overrides Misc.lower def move(self, *args): """Move an item TAGORID given in ARGS.""" @@ -3224,7 +3355,23 @@ def tag_raise(self, *args): (optional above another item).""" self.tk.call((self._w, 'raise') + args) - lift = tkraise = tag_raise + lift = tkraise = tag_raise # overrides Misc.tkraise + + def rchars(self, *args): + """Replace the text or coordinates between indices FIRST and LAST of + the items identified by TAGORID with STRING. + + Text items replace their text; line and polygon items replace their + coordinates, in which case STRING is a list of coordinates. Other + items ignore this operation.""" + self.tk.call((self._w, 'rchars') + args) + + def rotate(self, *args): # new in Tk 9.0 + """Rotate the coordinates of the items identified by TAGORID about the + origin (XORIGIN, YORIGIN) by ANGLE degrees anticlockwise. + + Negative values of ANGLE rotate clockwise.""" + self.tk.call((self._w, 'rotate') + args) def scale(self, *args): """Scale item TAGORID with XORIGIN, YORIGIN, XSCALE, YSCALE.""" @@ -3369,7 +3516,7 @@ def selection_adjust(self, index): select_adjust = selection_adjust - def selection_clear(self): + def selection_clear(self): # overrides Misc.selection_clear """Clear the selection if it is in this widget.""" self.tk.call(self._w, 'selection', 'clear') @@ -3401,6 +3548,14 @@ def selection_to(self, index): select_to = selection_to + def validate(self): + """Force an evaluation of the validation command. + + This evaluates the command given by the validatecommand option, + independently of the conditions specified by the validate option. + Return whether the value is considered valid.""" + return self.tk.getboolean(self.tk.call(self._w, 'validate')) + class Frame(Widget): """Frame widget which may contain other widgets and can have a 3D border.""" @@ -3463,7 +3618,7 @@ def activate(self, index): """Activate item identified by INDEX.""" self.tk.call(self._w, 'activate', index) - def bbox(self, index): + def bbox(self, index): # overrides Misc.bbox """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle which encloses the item identified by the given index.""" return self._getints(self.tk.call(self._w, 'bbox', index)) or None @@ -3519,7 +3674,7 @@ def selection_anchor(self, index): select_anchor = selection_anchor - def selection_clear(self, first, last=None): + def selection_clear(self, first, last=None): # overrides Misc.selection_clear """Clear the selection from FIRST to LAST (included).""" self.tk.call(self._w, 'selection', 'clear', first, last) @@ -3540,7 +3695,7 @@ def selection_set(self, first, last=None): select_set = selection_set - def size(self): + def size(self): # overrides Misc.size """Return the number of elements in the listbox.""" return self.tk.getint(self.tk.call(self._w, 'size')) @@ -3673,6 +3828,14 @@ def post(self, x, y): """Display a menu at position X,Y.""" self.tk.call(self._w, 'post', x, y) + def postcascade(self, index): + """Post the submenu of the cascade entry at INDEX, unposting any + previously posted submenu. + + Has no effect if INDEX does not name a cascade entry or if this menu + is not posted.""" + self.tk.call(self._w, 'postcascade', index) + def type(self, index): """Return the type of the menu item at INDEX.""" return self.tk.call(self._w, 'type', index) @@ -3856,7 +4019,7 @@ def __init__(self, master=None, cnf={}, **kw): """ Widget.__init__(self, master, 'text', cnf, kw) - def bbox(self, index): + def bbox(self, index): # overrides Misc.bbox """Return a tuple of (x,y,width,height) which gives the bounding box of the visible part of the character at the given index.""" return self._getints( @@ -3970,6 +4133,22 @@ def edit(self, *args): """ return self.tk.call(self._w, 'edit', *args) + def edit_canredo(self): + """Return whether redo is possible. + + Return True if redo is possible, i.e. when the redo stack is + not empty, and False otherwise. + """ + return self.tk.getboolean(self.edit("canredo")) + + def edit_canundo(self): + """Return whether undo is possible. + + Return True if undo is possible, i.e. when the undo stack is + not empty, and False otherwise. + """ + return self.tk.getboolean(self.edit("canundo")) + def edit_modified(self, arg=None): """Get or Set the modified flag @@ -4041,7 +4220,7 @@ def image_create(self, index, cnf={}, **kw): self._w, "image", "create", index, *self._options(cnf, kw)) - def image_names(self): + def image_names(self): # overrides Misc.image_names """Return all names of embedded images in this widget.""" return self.tk.call(self._w, "image", "names") @@ -4094,6 +4273,44 @@ def peer_names(self): # new in Tk 8.5 the widget itself).""" return self.tk.splitlist(self.tk.call(self._w, 'peer', 'names')) + def pendingsync(self): # new in Tk 8.5 + """Return whether the line heights calculations are not up-to-date. + + Return True if the line heights calculations are not up-to-date, + and False otherwise. + """ + return self.tk.getboolean(self.tk.call(self._w, 'pendingsync')) + + def sync(self, command=None): # new in Tk 8.5 + """Control the synchronization of the view of the text widget. + + If command is not specified, immediately bring the line metrics + up-to-date by forcing computation of any outdated line heights. + The command returns immediately if there is no such outdated line. + + If command is specified, schedule it to be executed (by the event + loop) exactly once as soon as all line heights are up-to-date. + If there are no pending line metrics calculations, command is + executed immediately. + """ + if command is None: + self.tk.call(self._w, 'sync') + else: + def callit(): + try: + command() + finally: + try: + self.deletecommand(name) + except TclError: + pass + try: + callit.__name__ = command.__name__ + except AttributeError: + callit.__name__ = type(command).__name__ + name = self._register(callit) + self.tk.call(self._w, 'sync', '-command', name) + def replace(self, index1, index2, chars, *args): # new in Tk 8.5 """Replaces the range of characters between index1 and index2 with the given characters and tags specified by args. @@ -4305,20 +4522,23 @@ def __init__(self, master, variable, value, *values, **kwargs): """Construct an optionmenu widget with the parent MASTER, with the option textvariable set to VARIABLE, the initially selected value VALUE, the other menu values VALUES and an additional - keyword argument command.""" - kw = {"borderwidth": 2, "textvariable": variable, - "indicatoron": 1, "relief": RAISED, "anchor": "c", - "highlightthickness": 2, "name": kwargs.pop("name", None)} + keyword argument command. + + Other keyword arguments are passed to the underlying menubutton + and may override its default appearance.""" + name = kwargs.pop("name", None) + callback = kwargs.pop("command", None) + # Default appearance, which may be overridden by keyword arguments. + kw = {"borderwidth": 2, "indicatoron": 1, "relief": RAISED, + "anchor": "c", "highlightthickness": 2} + kw.update(kwargs) + # These options are controlled by OptionMenu itself. + kw["textvariable"] = variable + kw["name"] = name Widget.__init__(self, master, "menubutton", kw) self.widgetName = 'tk_optionMenu' menu = self.__menu = Menu(self, name="menu", tearoff=0) self.menuname = menu._w - # 'command' is the only supported keyword - callback = kwargs.get('command') - if 'command' in kwargs: - del kwargs['command'] - if kwargs: - raise TclError('unknown option -'+next(iter(kwargs))) menu.add_command(label=value, command=_setit(variable, value, callback)) for v in values: @@ -4415,6 +4635,13 @@ def blank(self): """Display a transparent image.""" self.tk.call(self.name, 'blank') + def redither(self): + """Recalculate the dithered image in each window where it is displayed. + + Useful when the image data was supplied in pieces, in which case the + dithered image may not be exactly correct.""" + self.tk.call(self.name, 'redither') + def cget(self, option): """Return the value of OPTION.""" return self.tk.call(self.name, 'cget', '-' + option) @@ -4529,26 +4756,59 @@ def copy_replace(self, sourceImage, *, from_coords=None, to=None, shrink=False, options.extend(('-compositingrule', compositingrule)) self.tk.call(self.name, 'copy', sourceImage, *options) - def get(self, x, y): - """Return the color (red, green, blue) of the pixel at X,Y.""" - return self.tk.call(self.name, 'get', x, y) + @staticmethod + def _metadata(metadata): + # A Tcl dict is a flat list of alternating keys and values. A Python + # dict passed directly would expand to its keys only, so flatten it. + flat = () + for key, value in metadata.items(): + flat += (key, value) + return flat + + def get(self, x, y, *, withalpha=False): + """Return the color of the pixel at X,Y as a tuple of its red, green + and blue components. + + If WITHALPHA is true, the returned tuple has a fourth element giving + the alpha (opacity) value of the pixel. This requires Tcl/Tk 9.0 or + newer. + """ + args = (self.name, 'get', x, y) + if withalpha: + args += ('-withalpha',) + return self.tk.call(args) - def put(self, data, to=None): + def put(self, data, to=None, *, format=None, metadata=None): """Put row formatted colors to image starting from - position TO, e.g. image.put("{red green} {blue yellow}", to=(4,6))""" + position TO, e.g. image.put("{red green} {blue yellow}", to=(4,6)) + + The FORMAT option specifies the format of the image DATA, so that only + image file format handlers whose names begin with it are tried. + + The METADATA option, a dictionary passed to the image format driver, + requires Tcl/Tk 9.0 or newer. + """ args = (self.name, 'put', data) + if format is not None: + args += ('-format', format) + if metadata is not None: + args += ('-metadata', self._metadata(metadata)) if to: if to[0] == '-to': to = to[1:] - args = args + ('-to',) + tuple(to) + args += ('-to',) + tuple(to) self.tk.call(args) - def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False): + def read(self, filename, format=None, *, from_coords=None, to=None, + shrink=False, metadata=None): """Reads image data from the file named FILENAME into the image. The FORMAT option specifies the format of the image data in the file. + The METADATA option, a dictionary passed to the image format driver, + requires Tcl/Tk 9.0 or newer. + The FROM_COORDS option specifies a rectangular sub-region of the image file data to be copied to the destination image. It must be a tuple or a list of 1 to 4 integers (x1, y1, x2, y2). (x1, y1) and @@ -4568,6 +4828,8 @@ def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False options = () if format is not None: options += ('-format', format) + if metadata is not None: + options += ('-metadata', self._metadata(metadata)) if from_coords is not None: options += ('-from', *from_coords) if shrink: @@ -4577,13 +4839,16 @@ def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False self.tk.call(self.name, 'read', filename, *options) def write(self, filename, format=None, from_coords=None, *, - background=None, grayscale=False): + background=None, grayscale=False, metadata=None): """Writes image data from the image to a file named FILENAME. The FORMAT option specifies the name of the image file format handler to be used to write the data to the file. If this option is not given, the format is guessed from the file extension. + The METADATA option, a dictionary passed to the image format driver, + requires Tcl/Tk 9.0 or newer. + The FROM_COORDS option specifies a rectangular region of the image to be written to the image file. It must be a tuple or a list of 1 to 4 integers (x1, y1, x2, y2). If only x1 and y1 are specified, @@ -4602,6 +4867,8 @@ def write(self, filename, format=None, from_coords=None, *, options = () if format is not None: options += ('-format', format) + if metadata is not None: + options += ('-metadata', self._metadata(metadata)) if from_coords is not None: options += ('-from', *from_coords) if grayscale: @@ -4611,9 +4878,12 @@ def write(self, filename, format=None, from_coords=None, *, self.tk.call(self.name, 'write', filename, *options) def data(self, format=None, *, from_coords=None, - background=None, grayscale=False): + background=None, grayscale=False, metadata=None): """Returns image data. + The METADATA option, a dictionary passed to the image format driver, + requires Tcl/Tk 9.0 or newer. + The FORMAT option specifies the name of the image file format handler to be used. If this option is not given, this method uses a format that consists of a tuple (one element per row) of strings @@ -4640,6 +4910,8 @@ def data(self, format=None, *, from_coords=None, options = () if format is not None: options += ('-format', format) + if metadata is not None: + options += ('-metadata', self._metadata(metadata)) if from_coords is not None: options += ('-from', *from_coords) if grayscale: @@ -4716,7 +4988,7 @@ def __init__(self, master=None, cnf={}, **kw): """ Widget.__init__(self, master, 'spinbox', cnf, kw) - def bbox(self, index): + def bbox(self, index): # overrides Misc.bbox """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle which encloses the character given by index. @@ -4825,7 +5097,7 @@ def selection_adjust(self, index): """ return self.selection("adjust", index) - def selection_clear(self): + def selection_clear(self): # overrides Misc.selection_clear """Clear the selection If the selection isn't in this widget then the @@ -4859,6 +5131,14 @@ def selection_to(self, index): """Set the variable end of a selection to INDEX.""" self.selection('to', index) + def validate(self): + """Force an evaluation of the validation command. + + This evaluates the command given by the validatecommand option, + independently of the conditions specified by the validate option. + Return whether the value is considered valid.""" + return self.tk.getboolean(self.tk.call(self._w, 'validate')) + ########################################################################### @@ -4922,7 +5202,7 @@ def remove(self, child): """ self.tk.call(self._w, 'forget', child) - forget = remove + forget = remove # overrides Pack.forget def identify(self, x, y): """Identify the panedwindow component at point x, y diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 4f9eb44f0346773..c61881d48724217 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -23,53 +23,202 @@ askstring -- get a string from the user """ -from tkinter import Button, Entry, Frame, Label, Message, Tk, Toplevel +import tkinter +from tkinter import Button, Label, Tk, Toplevel, TclError from tkinter import _get_temp_root, _destroy_temp_root from tkinter import messagebox -from tkinter.constants import ACTIVE, BOTH, END, LEFT, RIDGE, W, E +from tkinter import ttk +from tkinter.constants import * +import contextlib __all__ = ["SimpleDialog", "Dialog", "askinteger", "askfloat", "askstring"] +# The standard dialog icons, which tk::MessageBox draws with themed images +# instead of the classic monochrome bitmaps. +_ICON_IMAGES = { + 'error': '::tk::icons::error', + 'info': '::tk::icons::information', + 'question': '::tk::icons::question', + 'warning': '::tk::icons::warning', +} + + +# Based on the Tk ::tk_dialog procedure, themed like ::tk::MessageBox by +# default. class SimpleDialog: + def _widget(self, klass, master, **kw): + # Create a themed (ttk) or classic (tkinter) widget. + return getattr(ttk if self.use_ttk else tkinter, klass)(master, **kw) + def __init__(self, master, text='', buttons=[], default=None, cancel=None, - title=None, class_=None): - if class_: - self.root = Toplevel(master, class_=class_) - else: - self.root = Toplevel(master) - if title: - self.root.title(title) - self.root.iconname(title) - + title=None, class_=None, *, bitmap=None, detail='', + use_ttk=True): + # Use the themed (ttk) widgets (modelled on tk::MessageBox) by default, + # or the classic Tk widgets (tk_dialog) if use_ttk is false. + self.use_ttk = use_ttk + + # 1. Create the top-level window and divide it into top + # and bottom parts. + + class_ = class_ or 'Dialog' + self.root = Toplevel(master, class_=class_) + # The default value of the title is space (" ") not the empty string + # because for some window managers, a + # w.title("") + # causes the window title to be w._name instead of the empty string. + self.root.title(title or ' ') + self.root.iconname(class_) + toplevel = master.winfo_toplevel() + if toplevel.winfo_viewable(): + self.root.wm_transient(toplevel) _setup_dialog(self.root) - - self.message = Message(self.root, text=text, aspect=400) - self.message.pack(expand=1, fill=BOTH) - self.frame = Frame(self.root) - self.frame.pack() + if self.use_ttk: + self.root.configure( + background=ttk.Style(self.root).lookup('.', 'background')) + + bot = self._widget('Frame', self.root, name='bot') + top = self._widget('Frame', self.root, name='top') + # The classic dialog (tk_dialog) gives its frames a raised border on + # X11; the themed one (tk::MessageBox) does not. + if not self.use_ttk and self.root._windowingsystem == 'x11': + bot.configure(relief=RAISED, bd=1) + top.configure(relief=RAISED, bd=1) + bot.pack(side=BOTTOM, fill=BOTH) + top.pack(side=TOP, fill=BOTH, expand=1) + bot.grid_anchor(CENTER) + + # 2. Fill the top part with bitmap and message (use the option + # database for -wraplength and -font so that they can be + # overridden by the caller). + + master.option_add(f'*{class_}.msg.wrapLength', '3i', 'widgetDefault') + master.option_add(f'*{class_}.msg.font', 'TkCaptionFont', 'widgetDefault') + master.option_add(f'*{class_}.dtl.wrapLength', '3i', 'widgetDefault') + master.option_add(f'*{class_}.dtl.font', 'TkDefaultFont', 'widgetDefault') + + # tk::MessageBox and tk_dialog pad the top part differently. + pad = '2m' if self.use_ttk else '3m' + + # The bitmap is packed first to claim the whole left side; the message + # and detail stack on its right, as in tk::MessageBox. + if bitmap: + if self.root._windowingsystem == 'aqua' and bitmap == 'error': + bitmap = 'stop' + image = _ICON_IMAGES.get(bitmap) if self.use_ttk else None + if image is not None and self.root.winfo_depth() >= 4: + # tk::MessageBox draws the standard icons with themed images. + self.bitmap = ttk.Label(self.root, name='bitmap', image=image) + else: + # ttk.Label has no -bitmap option, so use a classic label. + self.bitmap = Label(self.root, name='bitmap', bitmap=bitmap) + # The themed dialog anchors the icon to the top (like the bitmap's + # "nw" sticky in tk::MessageBox); the classic one centers it. + anchor = N if self.use_ttk else CENTER + self.bitmap.pack(in_=top, side=LEFT, anchor=anchor, + padx=pad, pady=pad) + + self.message = self._widget('Label', self.root, name='msg', + justify=LEFT, text=text) + self.detail = None + if self.use_ttk: + # tk::MessageBox anchors the message to the top-left corner. + self.message.configure(anchor=NW) + # The message expands to fill the space, unless there is a detail + # message below it which takes the extra space instead (cf. + # tk::MessageBox). + self.message.pack(in_=top, side=TOP, expand=not detail, fill=BOTH, + padx=pad, pady=pad) + if detail: + self.detail = self._widget('Label', self.root, name='dtl', + justify=LEFT, text=detail) + if self.use_ttk: + self.detail.configure(anchor=NW) + self.detail.pack(in_=top, side=TOP, expand=1, fill=BOTH, + padx=pad, pady=(0, pad)) + + self.frame = bot self.num = default self.cancel = cancel self.default = default - self.root.bind('<Return>', self.return_event) - for num in range(len(buttons)): - s = buttons[num] - b = Button(self.frame, text=s, - command=(lambda self=self, num=num: self.done(num))) - if num == default: - b.config(relief=RIDGE, borderwidth=8) - b.pack(side=LEFT, fill=BOTH, expand=1) + + # 3. Create a row of buttons at the bottom of the dialog. Each entry + # of "buttons" is either a label, or a mapping of button options -- like + # the "[name ?-option value ...?]" button specs in tk::MessageBox. + + # tk::MessageBox and tk_dialog space the buttons differently. + padx, pady = ('3m', '2m') if self.use_ttk else ('7.5p', '3p') + self._buttons = [] + for i, but in enumerate(buttons): + opts = {'text': but} if isinstance(but, str) else dict(but) + b = self._widget('Button', self.root, name=f'button{i}', **opts) + # The dialog controls the command and the default ring, overriding + # anything set in the button options (cf. tk::MessageBox). + b.configure(command=(lambda self=self, i=i: self.done(i)), + default=ACTIVE if i == default else NORMAL) + # Alt + the underlined character (an "underline" button option) + # invokes the button (cf. tk::AmpWidget in tk::MessageBox). + b.bind('<<AltUnderlined>>', lambda e: e.widget.invoke()) + b.grid(in_=bot, column=i, row=0, sticky=EW, padx=padx, pady=pady) + # tk::MessageBox makes the buttons equal width; tk_dialog does not. + bot.grid_columnconfigure(i, uniform='buttons' if self.use_ttk else '') + # We boost the size of some Mac buttons for l&f + if self.root._windowingsystem == 'aqua': + if str(opts.get('text', '')).lower() in ('ok', 'cancel'): + bot.grid_columnconfigure(i, minsize=90) + b.grid_configure(pady=7) + self._buttons.append(b) + + # 4. Bind <Return> to invoke the focused button, or the default button + # if the focus is elsewhere. Unlike tk_dialog (which tracks the focus + # by rebinding <Return> on <<PrevWindow>>/<<NextWindow>>), this reads + # the live focus, like tk::MessageBox, so it also works with the mouse. + + def on_return(event): + if event.widget.winfo_class() in ('Button', 'TButton'): + event.widget.invoke() + else: + self.return_event(event) + self.root.bind('<Return>', on_return) + # Alt + an underlined character invokes the matching button (cf. + # ::tk::AltKeyInDialog, bound by tk::MessageBox). + self.root.bind('<Alt-Key>', self._alt_key) + # The default ring follows the keyboard focus among the buttons + # (cf. tk::MessageBox). + self.root.bind('<FocusIn>', lambda e: self._set_default(e.widget, ACTIVE)) + self.root.bind('<FocusOut>', lambda e: self._set_default(e.widget, NORMAL)) + + # 5. Bind <Destroy> to record the cancel index, in case the window is + # destroyed by something else (e.g. its parent being destroyed). + + def on_destroy(event): + self.num = cancel + self.root.quit() + self.root.bind('<Destroy>', on_destroy) + self.root.protocol('WM_DELETE_WINDOW', self.wm_delete_window) - self.root.transient(master) _place_window(self.root, master) def go(self): self.root.wait_visibility() - self.root.grab_set() - self.root.mainloop() - self.root.destroy() + if self.default is not None: + focus = self._buttons[self.default] + else: + focus = self.root + + with _temp_grab_focus(self.root, focus): + try: + self.root.mainloop() + finally: + try: + # It's possible that the window has already been destroyed, + # hence this "try/except". Delete the Destroy handler so that + # self.num doesn't get reset by it. + self.root.bind('<Destroy>', '') + except TclError: + pass return self.num def return_event(self, event): @@ -88,7 +237,20 @@ def done(self, num): self.num = num self.root.quit() + def _alt_key(self, event): + # Invoke the button whose accelerator matches the Alt key. + target = _find_alt_key_target(self.root, event.char) + if target is not None: + target.event_generate('<<AltUnderlined>>') + + def _set_default(self, widget, state): + # Set a button's default ring. + if widget.winfo_class() in ('Button', 'TButton'): + widget.configure(default=state) + +# A base class for custom dialogs, with a button box modelled on +# ::tk::MessageBox. class Dialog(Toplevel): '''Class to open dialogs. @@ -96,7 +258,19 @@ class Dialog(Toplevel): This class is intended as a base class for custom dialogs ''' - def __init__(self, parent, title = None): + def _widget(self, klass, master, **kw): + # Create a themed (ttk) or classic (tkinter) widget. + return getattr(ttk if self.use_ttk else tkinter, klass)(master, **kw) + + def _frame(self, name): + # The classic dialog (tk_dialog) gives its frames a raised border on + # X11; the themed one (tk::MessageBox) does not. + frame = self._widget('Frame', self, name=name) + if not self.use_ttk and self._windowingsystem == 'x11': + frame.configure(relief=RAISED, bd=1) + return frame + + def __init__(self, parent, title=None, *, use_ttk=False): '''Initialize a dialog. Arguments: @@ -104,12 +278,26 @@ def __init__(self, parent, title = None): parent -- a parent window (the application window) title -- the dialog title + + use_ttk -- use the classic Tk widgets (the default), or the themed + (ttk) widgets if true ''' + # Use the classic Tk widgets by default, for compatibility: the themed + # (ttk) widgets set a themed background that classic widgets added by a + # subclass in body() would not match. The query dialogs opt into ttk. + self.use_ttk = use_ttk + master = parent if master is None: master = _get_temp_root() - Toplevel.__init__(self, master) + Toplevel.__init__(self, master, class_='Dialog') + + if self.use_ttk: + # Use a single background colour for the whole dialog so that it + # blends with the ttk widgets (cf. tk::MessageBox). + self.configure( + background=ttk.Style(self).lookup('.', 'background')) self.withdraw() # remain invisible for now # If the parent is not viewable, don't @@ -118,8 +306,8 @@ def __init__(self, parent, title = None): if parent is not None and parent.winfo_viewable(): self.transient(parent) - if title: - self.title(title) + self.title(title or ' ') + self.iconname('Dialog') _setup_dialog(self) @@ -127,9 +315,9 @@ def __init__(self, parent, title = None): self.result = None - body = Frame(self) + body = self._frame('top') self.initial_focus = self.body(body) - body.pack(padx=5, pady=5) + body.pack(side=TOP, fill=BOTH, expand=1) self.buttonbox() @@ -140,12 +328,12 @@ def __init__(self, parent, title = None): _place_window(self, parent) - self.initial_focus.focus_set() - # wait for window to appear on screen before calling grab_set self.wait_visibility() - self.grab_set() - self.wait_window(self) + # Dialog destroys itself in ok()/cancel(), so let _temp_grab_focus + # save/restore the focus and grab without destroying the window. + with _temp_grab_focus(self, self.initial_focus, destroy=False): + self.wait_window(self) def destroy(self): '''Destroy the window''' @@ -171,17 +359,58 @@ def buttonbox(self): override if you do not want the standard buttons ''' - box = Frame(self) - - w = Button(box, text="OK", width=10, command=self.ok, default=ACTIVE) - w.pack(side=LEFT, padx=5, pady=5) - w = Button(box, text="Cancel", width=10, command=self.cancel) - w.pack(side=LEFT, padx=5, pady=5) - - self.bind("<Return>", self.ok) - self.bind("<Escape>", self.cancel) + box = self._frame('bot') + + # tk::MessageBox and tk_dialog space the buttons differently. + padx, pady = ('3m', '2m') if self.use_ttk else ('7.5p', '3p') + for i, (name, label, command) in enumerate( + (('ok', '&OK', self.ok), ('cancel', '&Cancel', self.cancel))): + # Create a button with an accelerator key marked by "&" in the text + # (cf. tk::AmpWidget). + text, underline = _underline_ampersand(label) + b = self._widget('Button', self, name=name, text=text, + underline=underline, command=command, + default=ACTIVE if name == 'ok' else NORMAL) + b.bind('<<AltUnderlined>>', lambda e: e.widget.invoke()) + b.grid(in_=box, column=i, row=0, sticky=EW, padx=padx, pady=pady) + # tk::MessageBox makes the buttons equal width; tk_dialog does not. + box.grid_columnconfigure(i, uniform='buttons' if self.use_ttk else '') + if self._windowingsystem == 'aqua': + box.grid_columnconfigure(i, minsize=90) + b.grid_configure(pady=7) + + # Alt + an underlined character invokes the matching button (cf. + # ::tk::AltKeyInDialog, bound by tk::MessageBox). + self.bind('<Alt-Key>', self._alt_key) + # The default ring follows the keyboard focus among the buttons + # (cf. tk::MessageBox). + self.bind('<FocusIn>', lambda e: self._set_default(e.widget, ACTIVE)) + self.bind('<FocusOut>', lambda e: self._set_default(e.widget, NORMAL)) + self.bind('<Return>', self._return_event) + self.bind('<Escape>', self.cancel) + + box.pack(side=BOTTOM, fill=BOTH) + box.grid_anchor(CENTER) + + def _set_default(self, widget, state): + # Set a button's default ring. + if widget.winfo_class() in ('Button', 'TButton'): + widget.configure(default=state) + + def _return_event(self, event): + # Invoke the focused button, or accept the dialog if the focus is + # elsewhere (e.g. in an entry). + widget = event.widget + if widget.winfo_class() in ('Button', 'TButton'): + widget.invoke() + else: + self.ok() - box.pack() + def _alt_key(self, event): + # Invoke the button whose accelerator matches the Alt key. + target = _find_alt_key_target(self, event.char) + if target is not None: + target.event_generate('<<AltUnderlined>>') # # standard button semantics @@ -201,10 +430,6 @@ def ok(self, event=None): self.cancel() def cancel(self, event=None): - - # put focus back to the parent window - if self.parent is not None: - self.parent.focus_set() self.destroy() # @@ -217,7 +442,7 @@ def validate(self): dialog is destroyed. By default, it always validates OK. ''' - return 1 # override + return True # override def apply(self): '''process the data @@ -230,40 +455,114 @@ def apply(self): # Place a toplevel window at the center of parent or screen -# It is a Python implementation of ::tk::PlaceWindow. +# This is a Python implementation of ::tk::PlaceWindow. +def _wm_dimension(w, command): + # tk::WMFrameWidth and tk::WMTitleHeight (added in Tk 9.1) return the size + # of the window manager decoration. They are 0 except in SDL2 builds of + # Tk, and are missing in older versions. + try: + return int(w.tk.call(command)) + except TclError: + return 0 + + def _place_window(w, parent=None): w.wm_withdraw() # Remain invisible while we figure out the geometry w.update_idletasks() # Actualize geometry information + screenwidth = w.winfo_screenwidth() + screenheight = w.winfo_screenheight() minwidth = w.winfo_reqwidth() minheight = w.winfo_reqheight() maxwidth = w.winfo_vrootwidth() maxheight = w.winfo_vrootheight() + # "wm geometry" operates in window manager coordinates and thus includes a + # possible decoration frame and the title bar. + framewidth = _wm_dimension(w, '::tk::WMFrameWidth') + titleheight = _wm_dimension(w, '::tk::WMTitleHeight') + constrain = False + if minwidth + 2*framewidth > screenwidth: + minwidth = screenwidth - 2*framewidth + constrain = True + if minheight + titleheight + framewidth > screenheight: + minheight = screenheight - titleheight - framewidth + constrain = True + if parent is not None and parent.winfo_ismapped(): + # Center the window over the parent (which must be mapped). x = parent.winfo_rootx() + (parent.winfo_width() - minwidth) // 2 y = parent.winfo_rooty() + (parent.winfo_height() - minheight) // 2 + # Make sure that the window is on the screen and does not cover the + # window manager decoration. vrootx = w.winfo_vrootx() vrooty = w.winfo_vrooty() - x = min(x, vrootx + maxwidth - minwidth) - x = max(x, vrootx) - y = min(y, vrooty + maxheight - minheight) - y = max(y, vrooty) + x = min(x, vrootx + maxwidth - minwidth - framewidth) + x = max(x, vrootx + framewidth) + y = min(y, vrooty + maxheight - minheight - framewidth) + y = max(y, vrooty + titleheight) if w._windowingsystem == 'aqua': # Avoid the native menu bar which sits on top of everything. - y = max(y, 22) + y = max(y, 22 + titleheight) else: - x = (w.winfo_screenwidth() - minwidth) // 2 - y = (w.winfo_screenheight() - minheight) // 2 + # Center the window on the screen. + x = (screenwidth - minwidth) // 2 + y = (screenheight - minheight) // 2 w.wm_maxsize(maxwidth, maxheight) - w.wm_geometry('+%d+%d' % (x, y)) + geometry = f'{minwidth}x{minheight}' if constrain else '' + geometry += '+%d+%d' % (x - framewidth, y - titleheight) + w.wm_geometry(geometry) w.wm_deiconify() # Become visible at the desired location +def _underline_ampersand(text): + # Like tk::UnderlineAmpersand: "&&" is a literal "&"; a single "&" marks + # the following character as the underlined accelerator. Return the text + # without the markers and the index of the accelerator (-1 if none). + chars = [] + underline = -1 + i = 0 + while i < len(text): + if text[i] == '&': + if text[i+1:i+2] == '&': + chars.append('&') + i += 2 + continue + if underline < 0: + underline = len(chars) + else: + chars.append(text[i]) + i += 1 + return ''.join(chars), underline + + +def _find_alt_key_target(widget, char): + # Like tk::FindAltKeyTarget: find the widget whose underlined character + # matches CHAR, searching the widget and its descendants. + if widget.winfo_class() in ('Button', 'Checkbutton', 'Label', 'Radiobutton', + 'TButton', 'TCheckbutton', 'TLabel', + 'TRadiobutton'): + try: + under = int(widget.cget('underline')) + except (TclError, ValueError): + under = -1 + text = str(widget.cget('text')) + if 0 <= under < len(text) and char.lower() == text[under].lower(): + return widget + for child in widget.winfo_children(): + target = _find_alt_key_target(child, char) + if target is not None: + return target + return None + + def _setup_dialog(w): if w._windowingsystem == "aqua": - w.tk.call("::tk::unsupported::MacWindowStyle", "style", - w, "moveableModal", "") + if w.info_patchlevel() >= (9, 1): + w.wm_attributes(stylemask='titled') + else: + w.tk.call('::tk::unsupported::MacWindowStyle', 'style', + w, 'moveableModal', '') elif w._windowingsystem == "x11": w.wm_attributes(type="dialog") @@ -275,7 +574,7 @@ class _QueryDialog(Dialog): def __init__(self, title, prompt, initialvalue=None, minvalue = None, maxvalue = None, - parent = None): + parent = None, *, use_ttk=True): self.prompt = prompt self.minvalue = minvalue @@ -283,7 +582,7 @@ def __init__(self, title, prompt, self.initialvalue = initialvalue - Dialog.__init__(self, parent, title) + Dialog.__init__(self, parent, title, use_ttk=use_ttk) def destroy(self): self.entry = None @@ -291,11 +590,17 @@ def destroy(self): def body(self, master): - w = Label(master, text=self.prompt, justify=LEFT) - w.grid(row=0, padx=5, sticky=W) + # Wrap a long prompt, like tk::MessageBox wraps its message at 3 inches. + w = self._widget('Label', master, anchor=NW, text=self.prompt, + justify=LEFT, wraplength='3i') + w.grid(in_=master, padx='2m', pady='2m', sticky=NSEW) - self.entry = Entry(master, name="entry") - self.entry.grid(row=1, padx=5, sticky=W+E) + self.entry = self._widget('Entry', master, name='entry') + self.entry.grid(row=1, in_=master, padx='2m', pady=(0, '2m'), sticky=NSEW) + master.grid_rowconfigure(1, weight=1) + # The prompt and entry expand to the full width, like tk::MessageBox + # gives weight to its message column. + master.grid_columnconfigure(0, weight=1) if self.initialvalue is not None: self.entry.insert(0, self.initialvalue) @@ -312,7 +617,7 @@ def validate(self): self.errormessage + "\nPlease try again", parent = self ) - return 0 + return False if self.minvalue is not None and result < self.minvalue: messagebox.showwarning( @@ -321,7 +626,7 @@ def validate(self): "Please try again." % self.minvalue, parent = self ) - return 0 + return False if self.maxvalue is not None and result > self.maxvalue: messagebox.showwarning( @@ -330,11 +635,11 @@ def validate(self): "Please try again." % self.maxvalue, parent = self ) - return 0 + return False self.result = result - return 1 + return True class _QueryInteger(_QueryDialog): @@ -415,30 +720,96 @@ def askstring(title, prompt, **kw): return d.result +@contextlib.contextmanager +def _temp_grab_focus(grab, focus=None, destroy=True): + old_focus = grab.focus_get() + old_grab = grab.grab_current() + if old_grab is not None and old_grab.winfo_exists(): + old_status = old_grab.grab_status() + else: + old_status = None + # The "grab" command will fail if another application + # already holds the grab. So catch it. + try: + grab.grab_set() + except TclError: + pass + try: + if focus is not None and focus.winfo_exists(): + focus.focus_set() + + yield + + finally: + if old_focus is not None: + try: + old_focus.focus_set() + except TclError: + pass + try: + grab.grab_release() + except TclError: + pass + if destroy: + try: + grab.destroy() + except TclError: + pass + if (old_grab is not None and old_grab.winfo_exists() + and old_grab.winfo_ismapped()): + # The "grab" command will fail if another application + # already holds the grab. So catch it. + try: + if old_status == 'global': + old_grab.grab_set_global() + else: + old_grab.grab_set() + except TclError: + pass + + if __name__ == '__main__': def test(): root = Tk() - def doit(root=root): + use_ttk = tkinter.BooleanVar(root, value=True) + + def test_dialog(): d = SimpleDialog(root, text="This is a test dialog. " "Would this have been an actual dialog, " "the buttons below would have been glowing " "in soft pink light.\n" "Do you believe this?", - buttons=["Yes", "No", "Cancel"], + buttons=[{'text': 'Yes', 'underline': 0}, + {'text': 'No', 'underline': 0}, + {'text': 'Cancel', 'underline': 0}], default=0, cancel=2, - title="Test Dialog") + title="Test Dialog", + bitmap='question', + detail="Alt+Y, Alt+N and Alt+C work too.", + use_ttk=use_ttk.get()) print(d.go()) - print(askinteger("Spam", "Egg count", initialvalue=12*12)) + + def test_integer(): + print(askinteger("Spam", "Egg count", initialvalue=12*12, + use_ttk=use_ttk.get())) + + def test_float(): print(askfloat("Spam", "Egg weight\n(in tons)", minvalue=1, - maxvalue=100)) - print(askstring("Spam", "Egg label")) - t = Button(root, text='Test', command=doit) - t.pack() - q = Button(root, text='Quit', command=t.quit) - q.pack() - t.mainloop() + maxvalue=100, use_ttk=use_ttk.get())) + + def test_string(): + print(askstring("Spam", "Egg label", use_ttk=use_ttk.get())) + + tkinter.Checkbutton(root, text='Use themed (ttk) widgets', + variable=use_ttk).pack(fill=X) + Button(root, text='SimpleDialog', command=test_dialog).pack(fill=X) + Button(root, text='askinteger', command=test_integer).pack(fill=X) + Button(root, text='askfloat', command=test_float).pack(fill=X) + Button(root, text='askstring', command=test_string).pack(fill=X) + Button(root, text='Quit', command=root.quit).pack(fill=X) + root.mainloop() test() diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index 5c5ef11ae05ba6e..0cabc5c3140fd6b 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -492,6 +492,20 @@ def theme_names(self): return self.tk.splitlist(self.tk.call(self._name, "theme", "names")) + def theme_styles(self, themename=None): + """Returns a list of all styles in themename. + + If themename is omitted, the current theme is used. + + Availability: Tk 9.0. + """ + if themename is None: + return self.tk.splitlist( + self.tk.call(self._name, "theme", "styles")) + return self.tk.splitlist( + self.tk.call(self._name, "theme", "styles", themename)) + + def theme_use(self, themename=None): """If themename is None, returns the theme in use, otherwise, set the current theme to themename, refreshes all widgets and emits @@ -646,7 +660,7 @@ def __init__(self, master=None, widget=None, **kw): Widget.__init__(self, master, widget or "ttk::entry", kw) - def bbox(self, index): + def bbox(self, index): # overrides Misc.bbox """Return a tuple of (x, y, width, height) which describes the bounding box of the character given by index.""" return self._getints(self.tk.call(self._w, "bbox", index)) @@ -824,7 +838,7 @@ def add(self, child, **kw): self.tk.call(self._w, "add", child, *(_format_optdict(kw))) - def forget(self, tab_id): + def forget(self, tab_id): # overrides Pack.forget """Removes the tab specified by tab_id, unmaps and unmanages the associated window.""" self.tk.call(self._w, "forget", tab_id) @@ -1187,7 +1201,7 @@ def __init__(self, master=None, **kw): Widget.__init__(self, master, "ttk::treeview", kw) - def bbox(self, item, column=None): + def bbox(self, item, column=None): # overrides Misc.bbox """Returns the bounding box (relative to the treeview widget's window) of the specified item in the form x y width height. @@ -1246,7 +1260,7 @@ def exists(self, item): return self.tk.getboolean(self.tk.call(self._w, "exists", item)) - def focus(self, item=None): + def focus(self, item=None): # overrides Misc.focus """If item is specified, sets the focus item to item. Otherwise, returns the current focus item, or '' if there is none.""" return self.tk.call(self._w, "focus", item) @@ -1377,7 +1391,9 @@ def move(self, item, parent, index): def next(self, item): """Returns the identifier of item's next sibling, or '' if item - is the last child of its parent.""" + is the last child of its parent. + + Equivalent to after_item(item, hidden=True, recurse=False).""" return self.tk.call(self._w, "next", item) @@ -1389,10 +1405,542 @@ def parent(self, item): def prev(self, item): """Returns the identifier of item's previous sibling, or '' if - item is the first child of its parent.""" + item is the first child of its parent. + + Equivalent to before_item(item, hidden=True, recurse=False).""" return self.tk.call(self._w, "prev", item) + def after_item(self, item, *, hidden=False, recurse=True): + """Return the identifier of the item after item, or '' if there is + none. + + The result can be the first child, a next sibling, or a next sibling + of an ancestor. By default only visible items are considered; if + hidden is true, hidden items are included too. If recurse is false, + only siblings of item are considered (next item). + + * Availability: Tk 9.1""" + options = [] + if hidden: + options.append("-hidden") + if not recurse: + options.append("-norecurse") + return self.tk.call(self._w, "after", *options, item) + + + def before_item(self, item, *, hidden=False, recurse=True): + """Return the identifier of the item before item, or '' if there is + none. + + The result can be the previous sibling or the parent of item. By + default only visible items are considered; if hidden is true, hidden + items are included too. If recurse is false, only siblings of item + are considered (previous item). + + * Availability: Tk 9.1""" + options = [] + if hidden: + options.append("-hidden") + if not recurse: + options.append("-norecurse") + return self.tk.call(self._w, "before", *options, item) + + + def depth(self, item): + """Return the number of levels between item and the root item. + + * Availability: Tk 9.1""" + return self.tk.getint(self.tk.call(self._w, "depth", item)) + + + def haschildren(self, item): + """Return True if item has children, False otherwise. + + * Availability: Tk 9.1""" + return self.tk.getboolean(self.tk.call(self._w, "haschildren", item)) + + + def visible(self, item): + """Return True if item is visible, False otherwise. + + An item is visible if it is not detached, not hidden, and all of its + ancestors are open and not hidden. + + * Availability: Tk 9.1""" + return self.tk.getboolean(self.tk.call(self._w, "visible", item)) + + + def size(self, item, *, hidden=False, recurse=False): # overrides Misc.size + """Return the number of children of item. + + If hidden is true, hidden items are included. If recurse is true, + all descendants of item are included. Use '' for the root item. + size(item, hidden=True) equals len(get_children(item)). + + * Availability: Tk 9.1""" + options = [] + if hidden: + options.append("-hidden") + if recurse: + options.append("-recurse") + return self.tk.getint(self.tk.call(self._w, "size", *options, item)) + + + def range(self, first, last, *, hidden=False, recurse=True): + """Return a tuple of items from first through last, inclusive. + + If hidden is true, hidden items are included. If recurse is false, + descendants and ancestors are excluded. + + * Availability: Tk 9.1""" + options = [] + if hidden: + options.append("-hidden") + if not recurse: + options.append("-norecurse") + return self.tk.splitlist( + self.tk.call(self._w, "range", *options, first, last)) + + + def identifier(self, item, index): + """Return the identifier of the item at index within item's list of + children. + + * Availability: Tk 9.1""" + return self.tk.call(self._w, "identifier", item, index) + + + def current(self): + """Return the current item id and column id as a 2-tuple, or an empty + tuple if there is none. + + The current item is the item under the mouse pointer. + + * Availability: Tk 9.1""" + return self.tk.splitlist(self.tk.call(self._w, "current")) + + + def _itemlist(self, command, items, recurse): + if len(items) == 1 and isinstance(items[0], (tuple, list)): + items = items[0] + options = ("-recurse",) if recurse else () + self.tk.call(self._w, command, *options, items) + + + def expand(self, *items, recurse=False): + """Set all of the specified items to the open state. + + If recurse is true, also open all of their descendants; this requires + Tk 9.1. Use '' for the root item. + + expand(item) is equivalent to item(item, open=True).""" + try: + self._itemlist("expand", items, recurse) + except tkinter.TclError: + if recurse or self.info_patchlevel() >= (9, 1): + raise + self._open(True, items) + + + def collapse(self, *items, recurse=False): + """Set all of the specified items to the closed state. + + If recurse is true, also close all of their descendants; this requires + Tk 9.1. Use '' for the root item. + + collapse(item) is equivalent to item(item, open=False).""" + try: + self._itemlist("collapse", items, recurse) + except tkinter.TclError: + if recurse or self.info_patchlevel() >= (9, 1): + raise + self._open(False, items) + + + def _open(self, opening, items): + if len(items) == 1 and isinstance(items[0], (tuple, list)): + items = items[0] + for item in items: + if item != '': # the root item has no open state + self.item(item, open=opening) + + + def hide(self, *items, recurse=False): + """Hide all of the specified items and all of their child items. + + If recurse is true, also hide all of their descendants. Use '' for + the root item. hide(item) is equivalent to item(item, hidden=True). + + * Availability: Tk 9.1""" + self._itemlist("hide", items, recurse) + + + def unhide(self, *items, recurse=False): + """Unhide all of the specified items. + + If recurse is true, also unhide all of their descendants. Use '' for + the root item. unhide(item) is equivalent to item(item, hidden=False). + + * Availability: Tk 9.1""" + self._itemlist("unhide", items, recurse) + + + def detached(self, item=None): + """Return all detached items, or whether item is detached. + + Without arguments, return a tuple of all detached items (but not + their descendants; see detached_all). With item, return whether item + is detached; since Tk 9.1, also return true if an ancestor of item + is detached. + + * Availability: Tk 9.0""" + if item is None: + return self.tk.splitlist(self.tk.call(self._w, "detached")) + return self.tk.getboolean(self.tk.call(self._w, "detached", item)) + + + def detached_all(self): + """Return a tuple of all detached items and all of their descendants. + + * Availability: Tk 9.1""" + return self.tk.splitlist(self.tk.call(self._w, "detached", "-all")) + + + def cellfocus(self, cell=None): + """Get or set the focus cell. + + Without cell, return the focus cell as an (item, column) 2-tuple, or + an empty tuple if there is none. With cell, set the focus cell; use + '' to clear it. A cell is specified as an (item, column) pair. + + * Availability: Tk 9.1""" + if cell is None: + return self.tk.splitlist(self.tk.call(self._w, "cellfocus")) + return self.tk.call(self._w, "cellfocus", cell) + + + def sort(self, parent, *, column=None, command=None, dictionary=False, + integer=False, real=False, nocase=False, decreasing=False, + ignoreempty=False, recurse=False): + """Sort the children of parent. + + By default the children are sorted by the value of the first display + column, as Unicode strings, in increasing order. column selects the + column to sort on. dictionary, integer and real select the + comparison type; nocase makes string comparison case-insensitive. + command is a function of two values returning a negative, zero or + positive number. decreasing reverses the order. ignoreempty skips + empty values (with integer or real). recurse also sorts all + descendants. + + * Availability: Tk 9.1""" + options = [] + if column is not None: + options += ("-column", column) + if dictionary: + options.append("-dictionary") + if integer: + options.append("-integer") + if real: + options.append("-real") + if nocase: + options.append("-nocase") + if decreasing: + options.append("-decreasing") + if ignoreempty: + options.append("-ignoreempty") + if recurse: + options.append("-recurse") + if command is None: + self.tk.call(self._w, "sort", parent, *options) + else: + cmd = self.register(command) + try: + self.tk.call(self._w, "sort", parent, *options, + "-command", cmd) + finally: + self.deletecommand(cmd) + + + def search(self, parent, pattern, *, columns=None, start=None, stop=None, + dictionary=False, integer=False, real=False, nocase=False, + glob=False, regexp=False, backwards=False, hidden=False, + recurse=False, wraparound=False): + """Search parent's children for pattern and return the first match. + + By default pattern is matched for exact equality against the value of + each displayed column, as Unicode strings, searching forwards through + the direct children of parent. glob or regexp select glob-style or + regular expression matching; dictionary, integer and real select the + comparison type; nocase makes it case-insensitive. columns limits the + search to the given columns. start and stop bound the search; + backwards reverses its direction; wraparound continues from the other + end. hidden also searches hidden and closed items; recurse searches + all descendants. + + Return the identifier of the first matching item, or '' if there is no + match. See search_all (all matching items), search_cell (the first + matching cell) and search_all_cells (all matching cells). + + * Availability: Tk 9.1""" + return self._search(False, False, parent, pattern, columns, start, + stop, dictionary, integer, real, nocase, glob, + regexp, backwards, hidden, recurse, wraparound) + + + def search_all(self, parent, pattern, *, columns=None, start=None, + stop=None, dictionary=False, integer=False, real=False, + nocase=False, glob=False, regexp=False, backwards=False, + hidden=False, recurse=False, wraparound=False): + """Search parent's children for pattern and return all matches. + + Like search, but returns a tuple of the identifiers of all matching + items. + + * Availability: Tk 9.1""" + return self._search(True, False, parent, pattern, columns, start, + stop, dictionary, integer, real, nocase, glob, + regexp, backwards, hidden, recurse, wraparound) + + + def search_cell(self, parent, pattern, *, columns=None, start=None, + stop=None, dictionary=False, integer=False, real=False, + nocase=False, glob=False, regexp=False, backwards=False, + hidden=False, recurse=False, wraparound=False): + """Search parent's children for pattern and return the first cell. + + Like search, but matches and returns a cell, as an (item, column) + 2-tuple, or () if there is no match. + + * Availability: Tk 9.1""" + return self._search(False, True, parent, pattern, columns, start, + stop, dictionary, integer, real, nocase, glob, + regexp, backwards, hidden, recurse, wraparound) + + + def search_all_cells(self, parent, pattern, *, columns=None, start=None, + stop=None, dictionary=False, integer=False, + real=False, nocase=False, glob=False, regexp=False, + backwards=False, hidden=False, recurse=False, + wraparound=False): + """Search parent's children for pattern and return all matching cells. + + Like search, but returns a tuple of all matching cells, each an + (item, column) 2-tuple. + + * Availability: Tk 9.1""" + return self._search(True, True, parent, pattern, columns, start, + stop, dictionary, integer, real, nocase, glob, + regexp, backwards, hidden, recurse, wraparound) + + + def _search(self, all, cell, parent, pattern, columns, start, stop, + dictionary, integer, real, nocase, glob, regexp, backwards, + hidden, recurse, wraparound): + options = [] + if columns is not None: + options += ("-columns", columns) + if start is not None: + options += ("-start", start) + if stop is not None: + options += ("-stop", stop) + if dictionary: + options.append("-dictionary") + if integer: + options.append("-integer") + if real: + options.append("-real") + if nocase: + options.append("-nocase") + if glob: + options.append("-glob") + if regexp: + options.append("-regexp") + if backwards: + options.append("-backwards") + if hidden: + options.append("-hidden") + if recurse: + options.append("-recurse") + if wraparound: + options.append("-wraparound") + if cell: + options.append("-cell") + if all: + options.append("-all") + res = self.tk.call(self._w, "search", parent, *options, pattern) + if cell: + cells = tuple(self.tk.splitlist(c) + for c in self.tk.splitlist(res)) + if all: + return cells + return cells[0] if cells else () + if all: + return self.tk.splitlist(res) + return res + + + @staticmethod + def _cells(cells): + # Accept either several (item, column) cells or a single list of them. + if (len(cells) == 1 and cells[0] + and isinstance(cells[0], (tuple, list)) + and isinstance(cells[0][0], (tuple, list))): + return cells[0] + return cells + + + def cellselection(self): + """Return a tuple of the selected cells. + + Each cell is an (item, column) 2-tuple. The cell selection is + independent from the item selection (see selection). + + * Availability: Tk 9.1""" + return tuple(self.tk.splitlist(c) for c in + self.tk.splitlist(self.tk.call(self._w, "cellselection"))) + + + def _cellselection(self, selop, cells): + self.tk.call(self._w, "cellselection", selop, self._cells(cells)) + + + def cellselection_set(self, *cells): + """The specified cells become the new cell selection. + + Each cell is an (item, column) pair. Call without arguments to clear + the cell selection. + + * Availability: Tk 9.1""" + self._cellselection("set", cells) + + + def cellselection_add(self, *cells): + """Add the specified cells to the cell selection. + + * Availability: Tk 9.1""" + self._cellselection("add", cells) + + + def cellselection_remove(self, *cells): + """Remove the specified cells from the cell selection. + + * Availability: Tk 9.1""" + self._cellselection("remove", cells) + + + def _cellselection_range(self, selop, first, last, hidden, recurse): + options = [] + if not hidden: + options.append("-nohidden") + if not recurse: + options.append("-norecurse") + self.tk.call(self._w, "cellselection", selop, *options, first, last) + + + def cellselection_set_range(self, first, last, *, hidden=True, + recurse=True): + """Set the cell selection to the rectangle of cells from first to last. + + first and last are the opposite corner cells, each an (item, column) + pair, and must be in displayed columns. All other cells are + unselected. If hidden is false, hidden cells are excluded; if recurse + is false, cells in descendant items are excluded. + + * Availability: Tk 9.1""" + self._cellselection_range("set", first, last, hidden, recurse) + + + def cellselection_add_range(self, first, last, *, hidden=True, + recurse=True): + """Add the rectangle of cells from first to last to the cell selection. + + Like cellselection_set_range, but adds to the selection instead of + replacing it. + + * Availability: Tk 9.1""" + self._cellselection_range("add", first, last, hidden, recurse) + + + def cellselection_remove_range(self, first, last, *, hidden=True, + recurse=True): + """Remove the rectangle of cells from first to last from the selection. + + Like cellselection_set_range, but removes the cells from the selection. + + * Availability: Tk 9.1""" + self._cellselection_range("remove", first, last, hidden, recurse) + + + def cellselection_anchor(self, cell=None): + """Get or set the cell selection anchor. + + Without cell, return the anchor as an (item, column) 2-tuple, or an + empty tuple if unset. With cell, set the anchor; use '' to unset it. + + * Availability: Tk 9.1""" + if cell is None: + return self.tk.splitlist( + self.tk.call(self._w, "cellselection", "anchor")) + return self.tk.call(self._w, "cellselection", "anchor", cell) + + + def cellselection_includes(self, *cells): + """Return whether all of the specified cells are selected. + + * Availability: Tk 9.1""" + return self.tk.getboolean(self.tk.call( + self._w, "cellselection", "includes", self._cells(cells))) + + + def cellselection_present(self): + """Return whether any cell is selected. + + * Availability: Tk 9.1""" + return self.tk.getboolean( + self.tk.call(self._w, "cellselection", "present")) + + + def tag_cell_add(self, tagname, *cells): + """Add the given tag to each of the specified cells. + + Each cell is an (item, column) pair. Cell tags are independent from + item tags (see tag_add). + + * Availability: Tk 9.1""" + self.tk.call(self._w, "tag", "cell", "add", tagname, self._cells(cells)) + + + def tag_cell_remove(self, tagname, *cells): + """Remove the given tag from each of the specified cells. + + If no cell is specified, the tag is removed from all cells. + + * Availability: Tk 9.1""" + if cells: + self.tk.call(self._w, "tag", "cell", "remove", tagname, + self._cells(cells)) + else: + # Omit the cell list entirely (an empty list would match no cell). + self.tk.call(self._w, "tag", "cell", "remove", tagname) + + + def tag_cell_has(self, tagname, cell=None): + """Test for a cell tag, or list the cells that have it. + + If cell is specified, return whether that cell has the given tag. + Otherwise return a tuple of all cells (as (item, column) 2-tuples) + that have the tag. + + * Availability: Tk 9.1""" + if cell is None: + return tuple(self.tk.splitlist(c) for c in self.tk.splitlist( + self.tk.call(self._w, "tag", "cell", "has", tagname))) + return self.tk.getboolean( + self.tk.call(self._w, "tag", "cell", "has", tagname, cell)) + + def see(self, item): """Ensure that item is visible. diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 52cf3f0b7ccaa9c..3ea6c5b986ea78a 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -87,7 +87,8 @@ def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. - _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr'] + _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'd', 'br', 'fr', 'tr', + 'bd', 'rd', 'fd', 'td', 'brd', 'frd', 'trd'] # if we add binary f-strings, add: ['fb', 'fbr'] result = {''} for prefix in _valid_string_prefixes: @@ -578,7 +579,7 @@ def error(message, filename=None, location=None): parser = argparse.ArgumentParser() parser.add_argument(dest='filename', nargs='?', metavar='filename.py', - help='the file to tokenize; defaults to stdin') + help='the file to tokenize; defaults to `stdin`') parser.add_argument('-e', '--exact', dest='exact', action='store_true', help='display token names using the exact type') args = parser.parse_args(args) diff --git a/Lib/tomllib/mypy.ini b/Lib/tomllib/mypy.ini index 1761dce45562a60..f7eeffd575c1c76 100644 --- a/Lib/tomllib/mypy.ini +++ b/Lib/tomllib/mypy.ini @@ -12,6 +12,4 @@ pretty = True # Enable most stricter settings enable_error_code = ignore-without-code strict = True -strict_bytes = True -local_partial_types = True warn_unreachable = True diff --git a/Lib/trace.py b/Lib/trace.py index cd3a6d30661da39..43ec201c4696d1d 100644 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -604,7 +604,7 @@ def results(self): def main(): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('--version', action='version', version='trace 2.0') grp = parser.add_argument_group('Main options', @@ -612,27 +612,27 @@ def main(): grp.add_argument('-c', '--count', action='store_true', help='Count the number of times each line is executed and write ' - 'the counts to <module>.cover for each module executed, in ' - 'the module\'s directory. See also --coverdir, --file, ' - '--no-report below.') + 'the counts to `<module>.cover` for each module executed, in ' + 'the module\'s directory. See also `--coverdir`, `--file`, ' + '`--no-report` below.') grp.add_argument('-t', '--trace', action='store_true', - help='Print each line to sys.stdout before it is executed') + help='Print each line to `sys.stdout` before it is executed') grp.add_argument('-l', '--listfuncs', action='store_true', help='Keep track of which functions are executed at least once ' - 'and write the results to sys.stdout after the program exits. ' - 'Cannot be specified alongside --trace or --count.') + 'and write the results to `sys.stdout` after the program exits. ' + 'Cannot be specified alongside `--trace` or `--count`.') grp.add_argument('-T', '--trackcalls', action='store_true', help='Keep track of caller/called pairs and write the results to ' - 'sys.stdout after the program exits.') + '`sys.stdout` after the program exits.') grp = parser.add_argument_group('Modifiers') _grp = grp.add_mutually_exclusive_group() _grp.add_argument('-r', '--report', action='store_true', help='Generate a report from a counts file; does not execute any ' - 'code. --file must specify the results file to read, which ' - 'must have been created in a previous run with --count ' - '--file=FILE') + 'code. `--file` must specify the results file to read, which ' + 'must have been created in a previous run with `--count` ' + '`--file=FILE`') _grp.add_argument('-R', '--no-report', action='store_true', help='Do not generate the coverage report files. ' 'Useful if you want to accumulate over several runs.') @@ -641,14 +641,14 @@ def main(): help='File to accumulate counts over several runs') grp.add_argument('-C', '--coverdir', help='Directory where the report files go. The coverage report ' - 'for <package>.<module> will be written to file ' - '<dir>/<package>/<module>.cover') + 'for `<package>.<module>` will be written to file ' + '`<dir>/<package>/<module>.cover`') grp.add_argument('-m', '--missing', action='store_true', help='Annotate executable lines that were not executed with ' '">>>>>> "') grp.add_argument('-s', '--summary', action='store_true', - help='Write a brief summary for each file to sys.stdout. ' - 'Can only be used with --count or --report') + help='Write a brief summary for each file to `sys.stdout`. ' + 'Can only be used with `--count` or `--report`') grp.add_argument('-g', '--timing', action='store_true', help='Prefix each line with the time since the program started. ' 'Only used while tracing') @@ -661,7 +661,7 @@ def main(): 'module names.') grp.add_argument('--ignore-dir', action='append', default=[], help='Ignore files in the given directory ' - '(multiple directories can be joined by os.pathsep).') + '(multiple directories can be joined by `os.pathsep`).') parser.add_argument('--module', action='store_true', default=False, help='Trace a module. ') diff --git a/Lib/traceback.py b/Lib/traceback.py index 343d0e5f108c352..dcdab1f12e9a168 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -16,9 +16,9 @@ import io import importlib.util import pathlib -import _colorize from contextlib import suppress +lazy import _colorize try: from _missing_stdlib_info import _MISSING_STDLIB_MODULE_MESSAGES @@ -32,6 +32,36 @@ 'FrameSummary', 'StackSummary', 'TracebackException', 'walk_stack', 'walk_tb', 'print_list'] + +class _ShutdownTheme: + """Empty stand-in if `_colorize` cannot be imported during late shutdown.""" + def __getattr__(self, _): return self + def __getitem__(self, _): return "" + def __format__(self, _): return "" + def __str__(self): return "" + def __add__(self, other): return other + __radd__ = __add__ + + +_shutdown_theme = _ShutdownTheme() + + +def _safe_get_theme(*, force_color=False, force_no_color=False): + try: + return _colorize.get_theme( + force_color=force_color, force_no_color=force_no_color + ) + except ImportError: + return _shutdown_theme + + +def _safe_can_colorize(*, file=None): + try: + return _colorize.can_colorize(file=file) + except ImportError: + return False + + # # Formatting and printing lists of traceback lines. # @@ -85,10 +115,10 @@ def extract_tb(tb, limit=None): This is useful for alternate formatting of stack traces. If 'limit' is omitted or None, all entries are extracted. A pre-processed stack trace entry is a FrameSummary object - containing attributes filename, lineno, name, and line - representing the information that is usually printed for a stack - trace. The line is a string with leading and trailing - whitespace stripped; if the source is not available it is None. + representing the information that is usually printed for a + stack trace. The line attribute is a string with + leading and trailing whitespace stripped; if the source is not + available the corresponding attribute is None. """ return StackSummary._extract_from_extended_frame_gen( _walk_tb_with_full_positions(tb), limit=limit) @@ -151,7 +181,7 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ def _print_exception_bltin(exc, file=None, /): if file is None: file = sys.stderr if sys.stderr is not None else sys.__stderr__ - colorize = _colorize.can_colorize(file=file) + colorize = _safe_can_colorize(file=file) return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) @@ -199,9 +229,9 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= valuestr = _safe_string(value, 'exception') end_char = "\n" if insert_final_newline else "" if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback if value is None or not valuestr: line = f"{theme.type}{etype}{theme.reset}{end_char}" else: @@ -265,9 +295,8 @@ def extract_stack(f=None, limit=None): The return value has the same format as for extract_tb(). The optional 'f' and 'limit' arguments have the same meaning as for - print_stack(). Each item in the list is a quadruple (filename, - line number, function name, text), and the entries are in order - from oldest to newest stack frame. + print_stack(). Each item in the list is a FrameSummary object, + and the entries are in order from oldest to newest stack frame. """ if f is None: f = sys._getframe().f_back @@ -295,7 +324,7 @@ class FrameSummary: active when the frame was captured. - :attr:`name` The name of the function or method that was executing when the frame was captured. - - :attr:`line` The text from the linecache module for the + - :attr:`line` The text from the linecache module for the line of code that was running when the frame was captured. - :attr:`locals` Either None if locals were not supplied, or a dict mapping the name to the repr() of the variable. @@ -555,9 +584,9 @@ def format_frame_summary(self, frame_summary, **kwargs): if frame_summary.filename.startswith("<stdin-") and frame_summary.filename.endswith('>'): filename = "<stdin>" if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback row.append( ' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( theme.filename, @@ -1023,7 +1052,7 @@ def _wlen(s: str) -> int: def _display_width(line, offset=None): - """Calculate the extra amount of width space the given source + """Calculate the amount of width space the given source code segment might take if it were to be displayed on a fixed width output device. Supports wide unicode characters and emojis.""" @@ -1104,7 +1133,7 @@ class TracebackException: def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): - # NB: we need to accept exc_traceback, exc_value, exc_traceback to + # NB: we need to accept exc_type, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. # Handle loops in __cause__ or __context__. @@ -1187,12 +1216,20 @@ def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, elif exc_type and issubclass(exc_type, AttributeError) and \ getattr(exc_value, "name", None) is not None: wrong_name = getattr(exc_value, "name", None) - suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) - if suggestion: - if suggestion.isascii(): - self._str += f". Did you mean '.{suggestion}' instead of '.{wrong_name}'?" - else: - self._str += f". Did you mean '.{suggestion}' ({suggestion!a}) instead of '.{wrong_name}' ({wrong_name!a})?" + # Check cross-language/wrong-type hints first (more specific), + # then fall back to Levenshtein distance suggestions. + hint = None + if hasattr(exc_value, 'obj'): + hint = _get_cross_language_hint(exc_value.obj, wrong_name) + if hint: + self._str += f". {hint}" + else: + suggestion = _compute_suggestion_error(exc_value, exc_traceback, wrong_name) + if suggestion: + if suggestion.isascii(): + self._str += f". Did you mean '.{suggestion}' instead of '.{wrong_name}'?" + else: + self._str += f". Did you mean '.{suggestion}' ({suggestion!a}) instead of '.{wrong_name}' ({wrong_name!a})?" elif exc_type and issubclass(exc_type, NameError) and \ getattr(exc_value, "name", None) is not None: wrong_name = getattr(exc_value, "name", None) @@ -1336,9 +1373,9 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): """ colorize = kwargs.get("colorize", False) if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback indent = 3 * _depth * ' ' if not self._have_exc_type: @@ -1448,11 +1485,22 @@ def _find_keyword_typos(self): # Limit the number of possible matches to try max_matches = 3 matches = [] + + hint = _get_cross_language_keyword_hint(wrong_name) + if hint: + matches.append(hint) if _suggestions is not None: - suggestion = _suggestions._generate_suggestions(keyword.kwlist, wrong_name) + suggestion = _suggestions._generate_suggestions(keyword.kwlist + keyword.softkwlist, wrong_name) if suggestion: matches.append(suggestion) - matches.extend(difflib.get_close_matches(wrong_name, keyword.kwlist, n=max_matches, cutoff=0.5)) + matches.extend( + difflib.get_close_matches( + wrong_name, + keyword.kwlist + keyword.softkwlist, + n=max_matches, + cutoff=0.5 + ) + ) matches = matches[:max_matches] for suggestion in matches: if not suggestion or suggestion == wrong_name: @@ -1486,9 +1534,9 @@ def _format_syntax_error(self, stype, **kwargs): # Show exactly where the problem was found. colorize = kwargs.get("colorize", False) if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback filename_suffix = '' if self.lineno is not None: yield ' File {}"{}"{}, line {}{}{}\n'.format( @@ -1689,6 +1737,77 @@ def print(self, *, file=None, chain=True, **kwargs): _MOVE_COST = 2 _CASE_COST = 1 +# Cross-language method suggestions for builtin types. +# Consulted as a fallback when Levenshtein-based suggestions find no match. +# +# Inclusion criteria: +# +# 1. Must have evidence of real cross-language confusion (Stack Overflow +# traffic, bug reports in production repos, developer survey data). +# 2. Must not be catchable by Levenshtein distance (too different from +# the correct Python method name). +# +# Each entry maps a wrong method name to a list of (type, suggestion, is_raw) +# tuples. The lookup checks isinstance() so subclasses are also matched. +# If is_raw is False, the suggestion is wrapped in "Did you mean '.X'?". +# If is_raw is True, the suggestion is rendered as-is. +# +# See https://github.com/python/cpython/issues/146406. +_CROSS_LANGUAGE_HINTS = frozendict({ + # list -- JavaScript/Ruby equivalents + "push": ((list, "append", False),), + "concat": ((list, "extend", False),), + # list -- Java/C# equivalents + "addAll": ((list, "extend", False),), + "contains": ((list, "Use 'x in list'.", True),), + # list -- wrong-type suggestion (user expected a set) + "add": ((list, "Did you mean to use a 'set' object?", True), + (frozenset, "Did you mean to use a 'set' object?", True)), + # str -- JavaScript equivalents + "toUpperCase": ((str, "upper", False),), + "toLowerCase": ((str, "lower", False),), + "trimStart": ((str, "lstrip", False),), + "trimEnd": ((str, "rstrip", False),), + # dict -- Java/JavaScript equivalents + "keySet": ((dict, "keys", False),), + "entrySet": ((dict, "items", False),), + "entries": ((dict, "items", False),), + "putAll": ((dict, "update", False),), + "put": ((dict, "Use d[k] = v.", True),), + # tuple -- mutable method on immutable type (user expected a list) + "append": ((tuple, "Did you mean to use a 'list' object?", True),), + "extend": ((tuple, "Did you mean to use a 'list' object?", True),), + "insert": ((tuple, "Did you mean to use a 'list' object?", True),), + "remove": ((tuple, "Did you mean to use a 'list' object?", True), + (frozenset, "Did you mean to use a 'set' object?", True)), + # frozenset -- mutable method on immutable type (user expected a set) + "discard": ((frozenset, "Did you mean to use a 'set' object?", True),), + # frozendict -- mutable method on immutable type (user expected a dict) + "update": ((frozenset, "Did you mean to use a 'set' object?", True), + (frozendict, "Did you mean to use a 'dict' object?", True)), + # clear() -- shared across immutable container types (user expected the mutable counterpart) + "clear": ((tuple, "Did you mean to use a 'list' object?", True), + (frozenset, "Did you mean to use a 'set' object?", True), + (frozendict, "Did you mean to use a 'dict' object?", True)), + # float -- bitwise operators belong to int + "__or__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__and__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__xor__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__lshift__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), + "__rshift__": ((float, "Did you mean to use an 'int' object? Bitwise operators are not supported by 'float'.", True),), +}) + + +# Cross-language keyword suggestions. +_CROSS_LANGUAGE_KEYWORD_HINTS = frozendict({ + # C/C++ equivalents + 'switch': 'match', + 'delete': 'del', + # function define equivalents + 'function': 'def', + 'func': 'def', + 'void': 'def', +}) def _substitution_cost(ch_a, ch_b): if ch_a == ch_b: @@ -1751,6 +1870,30 @@ def _check_for_nested_attribute(obj, wrong_name, attrs): return None +def _get_cross_language_hint(obj, wrong_name): + """Check if wrong_name is a common method name from another language, + a mutable method on an immutable type, or a method tried on None. + + Uses isinstance() so subclasses of builtin types also get hints. + Returns a formatted hint string, or None. + """ + entries = _CROSS_LANGUAGE_HINTS.get(wrong_name) + if entries is None: + return None + for check_type, hint, is_raw in entries: + if isinstance(obj, check_type): + if is_raw: + return hint + return f"Did you mean '.{hint}'?" + return None + + +def _get_cross_language_keyword_hint(wrong_name): + """Check if wrong_name is a common keyword from another language + """ + return _CROSS_LANGUAGE_KEYWORD_HINTS.get(wrong_name) + + def _get_safe___dir__(obj): # Use obj.__dir__() to avoid a TypeError when calling dir(obj). # See gh-131001 and gh-139933. diff --git a/Lib/types.py b/Lib/types.py index b4f9a5c5140860b..6c069591ab26ef0 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -195,18 +195,19 @@ class Baz(list[str]): ... class DynamicClassAttribute: """Route attribute access on a class to __getattr__. - This is a descriptor, used to define attributes that act differently when - accessed through an instance and through a class. Instance access remains - normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. - - This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name. (Enum used this between Python - versions 3.4 - 3.9 .) - - Subclass from this to use a different method of accessing virtual attributes - and still be treated properly by the inspect module. (Enum uses this since - Python 3.10 .) + This is a descriptor, used to define attributes that act differently + when accessed through an instance and through a class. Instance access + remains normal, but access to an attribute through a class will be + routed to the class's __getattr__ method; this is done by raising + AttributeError. + + This allows one to have properties active on an instance, and have + virtual attributes on the class with the same name. (Enum used this + between Python versions 3.4 - 3.9 .) + + Subclass from this to use a different method of accessing virtual + attributes and still be treated properly by the inspect module. (Enum + uses this since Python 3.10 .) """ def __init__(self, fget=None, fset=None, fdel=None, doc=None): diff --git a/Lib/typing.py b/Lib/typing.py index e7563a53878da50..1579f492003f748 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -5,7 +5,7 @@ * Generic, Protocol, and internal machinery to support generic aliases. All subscripted types like X[int], Union[int, str] are generic aliases. * Various "special forms" that have unique meanings in type annotations: - NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others. + Any, Never, ClassVar, Self, Concatenate, Unpack, and others. * Classes whose instances can be type arguments to generic classes and functions: TypeVar, ParamSpec, TypeVarTuple. * Public helper functions: get_type_hints, overload, cast, final, and others. @@ -28,6 +28,7 @@ import sys import types from types import GenericAlias +lazy import annotationlib from _typing import ( _idfunc, @@ -163,15 +164,6 @@ 'Unpack', ] -class _LazyAnnotationLib: - def __getattr__(self, attr): - global _lazy_annotationlib - import annotationlib - _lazy_annotationlib = annotationlib - return getattr(annotationlib, attr) - -_lazy_annotationlib = _LazyAnnotationLib() - def _type_convert(arg, module=None, *, allow_special_forms=False, owner=None): """For converting None to type(None), and strings to ForwardRef.""" @@ -255,7 +247,7 @@ def _type_repr(obj): if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return _lazy_annotationlib.type_repr(obj) + return annotationlib.type_repr(obj) def _collect_type_parameters( @@ -442,19 +434,6 @@ def _rebuild_generic_alias(alias: GenericAlias, args: tuple[object, ...]) -> Gen return t -def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: - import warnings - - depr_message = ( - f"Failing to pass a value to the 'type_params' parameter " - f"of {funcname!r} is deprecated, as it leads to incorrect behaviour " - f"when calling {funcname} on a stringified annotation " - f"that references a PEP 695 type parameter. " - f"It will be disallowed in Python 3.15." - ) - warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) - - def _eval_type(t, globalns, localns, type_params, *, recursive_guard=frozenset(), format=None, owner=None, parent_fwdref=None, prefer_fwd_module=False): """Evaluate all forward references in the given type t. @@ -463,7 +442,7 @@ def _eval_type(t, globalns, localns, type_params, *, recursive_guard=frozenset() recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ - if isinstance(t, _lazy_annotationlib.ForwardRef): + if isinstance(t, annotationlib.ForwardRef): # If the forward_ref has __forward_module__ set, evaluate() infers the globals # from the module, and it will probably pick better than the globals we have here. # We do this only for calls from get_type_hints() (which opts in through the @@ -612,12 +591,12 @@ def __repr__(self): class Any(metaclass=_AnyMeta): """Special type indicating an unconstrained type. - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. + - Any is assignable to every type. + - Any assumed to have all methods and attributes. + - All values are assignable to Any. Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance + static type checkers. At runtime, Any cannot be used with instance checks. """ @@ -736,7 +715,7 @@ class Starship: ClassVar accepts only types and cannot be further subscribed. - Note that ClassVar is not a class itself, and should not + Note that ClassVar is not a class itself, and cannot be used with isinstance() or issubclass(). """ item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) @@ -766,7 +745,7 @@ class FastConnector(Connection): @_SpecialForm def Optional(self, parameters): - """Optional[X] is equivalent to Union[X, None].""" + """Optional[X] is equivalent to X | None.""" arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] @@ -809,7 +788,7 @@ def open_helper(file: str, mode: MODE) -> str: def TypeAlias(self, parameters): """Special form for marking type aliases. - Use TypeAlias to indicate that an assignment should + TypeAlias can be used to indicate that an assignment should be recognized as a proper type alias definition by type checkers. @@ -1004,7 +983,7 @@ def _make_forward_ref(code, *, parent_fwdref=None, **kwargs): kwargs['module'] = parent_fwdref.__forward_module__ if parent_fwdref.__owner__ is not None: kwargs['owner'] = parent_fwdref.__owner__ - forward_ref = _lazy_annotationlib.ForwardRef(code, **kwargs) + forward_ref = annotationlib.ForwardRef(code, **kwargs) # For compatibility, eagerly compile the forwardref's code. forward_ref.__forward_code__ return forward_ref @@ -1039,18 +1018,18 @@ def evaluate_forward_ref( VALUE. """ - if format == _lazy_annotationlib.Format.STRING: + if format == annotationlib.Format.STRING: return forward_ref.__forward_arg__ if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref if format is None: - format = _lazy_annotationlib.Format.VALUE + format = annotationlib.Format.VALUE value = forward_ref.evaluate(globals=globals, locals=locals, type_params=type_params, owner=owner, format=format) - if (isinstance(value, _lazy_annotationlib.ForwardRef) - and format == _lazy_annotationlib.Format.FORWARDREF): + if (isinstance(value, annotationlib.ForwardRef) + and format == annotationlib.Format.FORWARDREF): return value if isinstance(value, str): @@ -1364,32 +1343,35 @@ def __dir__(self): class _GenericAlias(_BaseGenericAlias, _root=True): - # The type of parameterized generics. - # - # That is, for example, `type(List[int])` is `_GenericAlias`. - # - # Objects which are instances of this class include: - # * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. - # * Note that native container types, e.g. `tuple`, `list`, use - # `types.GenericAlias` instead. - # * Parameterized classes: - # class C[T]: pass - # # C[int] is a _GenericAlias - # * `Callable` aliases, generic `Callable` aliases, and - # parameterized `Callable` aliases: - # T = TypeVar('T') - # # _CallableGenericAlias inherits from _GenericAlias. - # A = Callable[[], None] # _CallableGenericAlias - # B = Callable[[T], None] # _CallableGenericAlias - # C = B[int] # _CallableGenericAlias - # * Parameterized `Final`, `ClassVar`, `TypeForm`, `TypeGuard`, and `TypeIs`: - # # All _GenericAlias - # Final[int] - # ClassVar[float] - # TypeForm[bytes] - # TypeGuard[bool] - # TypeIs[range] - + """The type of parameterized generics. + + That is, for example, `type(List[int])` is `_GenericAlias`. + + Objects which are instances of this class include: + * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. + * Note that native container types, e.g. `tuple`, `list`, use + `types.GenericAlias` instead. + * Parameterized classes: + class C[T]: pass + # C[int] is a _GenericAlias + * `Callable` aliases, generic `Callable` aliases, and + parameterized `Callable` aliases: + T = TypeVar('T') + # _CallableGenericAlias inherits from _GenericAlias. + A = Callable[[], None] # _CallableGenericAlias + B = Callable[[T], None] # _CallableGenericAlias + C = B[int] # _CallableGenericAlias + * Parameterized `Final`, `ClassVar`, `TypeForm`, `TypeGuard`, and `TypeIs`: + # All _GenericAlias + Final[int] + ClassVar[float] + TypeForm[bytearray] + TypeGuard[bool] + TypeIs[range] + + Note that instances of this class are not classes (e.g by `inspect.isclass`), + even though they behave like them. + """ def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) if not isinstance(args, tuple): @@ -1421,20 +1403,21 @@ def __ror__(self, left): @_tp_cache def __getitem__(self, args): - # Parameterizes an already-parameterized object. - # - # For example, we arrive here doing something like: - # T1 = TypeVar('T1') - # T2 = TypeVar('T2') - # T3 = TypeVar('T3') - # class A(Generic[T1]): pass - # B = A[T2] # B is a _GenericAlias - # C = B[T3] # Invokes _GenericAlias.__getitem__ - # - # We also arrive here when parameterizing a generic `Callable` alias: - # T = TypeVar('T') - # C = Callable[[T], None] - # C[int] # Invokes _GenericAlias.__getitem__ + """Parameterizes an already-parameterized object. + + For example, we arrive here doing something like: + T1 = TypeVar('T1') + T2 = TypeVar('T2') + T3 = TypeVar('T3') + class A(Generic[T1]): pass + B = A[T2] # B is a _GenericAlias + C = B[T3] # Invokes _GenericAlias.__getitem__ + + We also arrive here when parameterizing a generic `Callable` alias: + T = TypeVar('T') + C = Callable[[T], None] + C[int] # Invokes _GenericAlias.__getitem__ + """ if self.__origin__ in (Generic, Protocol): # Can't subscript Generic[...] or Protocol[...]. @@ -1451,20 +1434,20 @@ def __getitem__(self, args): return r def _determine_new_args(self, args): - # Determines new __args__ for __getitem__. - # - # For example, suppose we had: - # T1 = TypeVar('T1') - # T2 = TypeVar('T2') - # class A(Generic[T1, T2]): pass - # T3 = TypeVar('T3') - # B = A[int, T3] - # C = B[str] - # `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. - # Unfortunately, this is harder than it looks, because if `T3` is - # anything more exotic than a plain `TypeVar`, we need to consider - # edge cases. - + """Determines new __args__ for __getitem__. + + For example, suppose we had: + T1 = TypeVar('T1') + T2 = TypeVar('T2') + class A(Generic[T1, T2]): pass + T3 = TypeVar('T3') + B = A[int, T3] + C = B[str] + `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. + Unfortunately, this is harder than it looks, because if `T3` is + anything more exotic than a plain `TypeVar`, we need to consider + edge cases. + """ params = self.__parameters__ # In the example above, this would be {T3: str} for param in params: @@ -1817,7 +1800,7 @@ class Movie(TypedDict): def foo(**kwargs: Unpack[Movie]): ... Note that there is only some runtime checking of this operator. Not - everything the runtime allows may be accepted by static type checkers. + everything the runtime allows is accepted by static type checkers. For more information, see PEPs 646 and 692. """ @@ -1891,8 +1874,8 @@ def _get_protocol_attrs(cls): annotations = base.__annotations__ except Exception: # Only go through annotationlib to handle deferred annotations if we need to - annotations = _lazy_annotationlib.get_annotations( - base, format=_lazy_annotationlib.Format.FORWARDREF + annotations = annotationlib.get_annotations( + base, format=annotationlib.Format.FORWARDREF ) for attr in (*base.__dict__, *annotations): if not attr.startswith('_abc_') and attr not in EXCLUDED_ATTRIBUTES: @@ -2140,8 +2123,8 @@ def _proto_hook(cls, other): try: annos = base.__annotations__ except Exception: - annos = _lazy_annotationlib.get_annotations( - base, format=_lazy_annotationlib.Format.FORWARDREF + annos = annotationlib.get_annotations( + base, format=annotationlib.Format.FORWARDREF ) if attr in annos: break @@ -2328,7 +2311,7 @@ def runtime_checkable(cls): Such protocol can be used with isinstance() and issubclass(). Raise TypeError if applied to a non-protocol class. This allows a simple-minded structural check very similar to - one trick ponies in collections.abc such as Iterable. + one-trick ponies in collections.abc such as Iterable. For example:: @@ -2398,8 +2381,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=None): """Return type hints for an object. - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals and recursively replaces all + This is often the same as annotationlib.get_annotations(obj) or obj.__annotations__, + but it handles forward references encoded as string literals and recursively replaces all 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations @@ -2428,14 +2411,14 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, """ if getattr(obj, '__no_type_check__', None): return {} - Format = _lazy_annotationlib.Format + Format = annotationlib.Format if format is None: format = Format.VALUE # Classes require a special treatment. if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): - ann = _lazy_annotationlib.get_annotations(base, format=format) + ann = annotationlib.get_annotations(base, format=format) if format == Format.STRING: hints.update(ann) continue @@ -2468,7 +2451,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, else: return {k: _strip_annotations(t) for k, t in hints.items()} - hints = _lazy_annotationlib.get_annotations(obj, format=format) + hints = annotationlib.get_annotations(obj, format=format) if ( not hints and not isinstance(obj, types.ModuleType) @@ -2611,7 +2594,7 @@ def get_args(tp): def is_typeddict(tp): - """Check if an annotation is a TypedDict class. + """Check if an object is a TypedDict class. For example:: @@ -2708,10 +2691,10 @@ def _overload_dummy(*args, **kwds): def overload(func): """Decorator for overloaded functions/methods. - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. - - For example:: + In a non-stub file, place two or more stub definitions for the same + function in a row, each decorated with @overload, followed + by an implementation. The implementation should *not* + be decorated with @overload:: @overload def utf8(value: None) -> None: ... @@ -2719,10 +2702,11 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... + def utf8(value): + ... # implementation goes here - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload:: + In a stub file or in an abstract method (for example, in a Protocol definition), + the implementation may be omitted:: @overload def utf8(value: None) -> None: ... @@ -2730,8 +2714,6 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... - def utf8(value): - ... # implementation goes here The overloads for a function can be retrieved at runtime using the get_overloads() function. @@ -2767,7 +2749,7 @@ def final(f): """Decorator to indicate final methods and final classes. Use this decorator to indicate to type checkers that the decorated - method cannot be overridden, and decorated class cannot be subclassed. + method cannot be overridden, and the decorated class cannot be subclassed. For example:: @@ -2832,7 +2814,7 @@ class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. -# Internal type variable used for Type[]. +# Internal type bound to class object types. CT_co = TypeVar('CT_co', covariant=True, bound=type) @@ -2920,7 +2902,7 @@ class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - def new_user[U](user_class: Type[U]) -> U: + def new_user[U](user_class: type[U]) -> U: user = user_class() # (Here we could write the user object to a database) return user @@ -2933,7 +2915,7 @@ def new_user[U](user_class: Type[U]) -> U: @runtime_checkable class SupportsInt(Protocol): - """An ABC with one abstract method __int__.""" + """A protocol with one abstract method __int__.""" __slots__ = () @@ -2944,7 +2926,7 @@ def __int__(self) -> int: @runtime_checkable class SupportsFloat(Protocol): - """An ABC with one abstract method __float__.""" + """A protocol with one abstract method __float__.""" __slots__ = () @@ -2955,7 +2937,7 @@ def __float__(self) -> float: @runtime_checkable class SupportsComplex(Protocol): - """An ABC with one abstract method __complex__.""" + """A protocol with one abstract method __complex__.""" __slots__ = () @@ -2966,7 +2948,7 @@ def __complex__(self) -> complex: @runtime_checkable class SupportsBytes(Protocol): - """An ABC with one abstract method __bytes__.""" + """A protocol with one abstract method __bytes__.""" __slots__ = () @@ -2977,7 +2959,7 @@ def __bytes__(self) -> bytes: @runtime_checkable class SupportsIndex(Protocol): - """An ABC with one abstract method __index__.""" + """A protocol with one abstract method __index__.""" __slots__ = () @@ -2988,7 +2970,7 @@ def __index__(self) -> int: @runtime_checkable class SupportsAbs[T](Protocol): - """An ABC with one abstract method __abs__ that is covariant in its return type.""" + """A protocol with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @@ -2999,7 +2981,7 @@ def __abs__(self) -> T: @runtime_checkable class SupportsRound[T](Protocol): - """An ABC with one abstract method __round__ that is covariant in its return type.""" + """A protocol with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @@ -3020,10 +3002,10 @@ def _make_eager_annotate(types): for key, val in types.items()} def annotate(format): match format: - case _lazy_annotationlib.Format.VALUE | _lazy_annotationlib.Format.FORWARDREF: + case annotationlib.Format.VALUE | annotationlib.Format.FORWARDREF: return checked_types - case _lazy_annotationlib.Format.STRING: - return _lazy_annotationlib.annotations_to_string(types) + case annotationlib.Format.STRING: + return annotationlib.annotations_to_string(types) case _: raise NotImplementedError(format) return annotate @@ -3053,9 +3035,9 @@ def __new__(cls, typename, bases, ns): types = ns["__annotations__"] field_names = list(types) annotate = _make_eager_annotate(types) - elif (original_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: - types = _lazy_annotationlib.call_annotate_function( - original_annotate, _lazy_annotationlib.Format.FORWARDREF) + elif (original_annotate := annotationlib.get_annotate_from_class_namespace(ns)) is not None: + types = annotationlib.call_annotate_function( + original_annotate, annotationlib.Format.FORWARDREF) field_names = list(types) # For backward compatibility, type-check all the types at creation time @@ -3063,9 +3045,9 @@ def __new__(cls, typename, bases, ns): _type_check(typ, "field annotation must be a type") def annotate(format): - annos = _lazy_annotationlib.call_annotate_function( + annos = annotationlib.call_annotate_function( original_annotate, format) - if format != _lazy_annotationlib.Format.STRING: + if format != annotationlib.Format.STRING: return {key: _type_check(val, f"field {key} annotation must be a type") for key, val in annos.items()} return annos @@ -3116,7 +3098,7 @@ def annotate(format): def NamedTuple(typename, fields, /): - """Typed version of namedtuple. + """Typed version of collections.namedtuple. Usage:: @@ -3128,8 +3110,8 @@ class Employee(NamedTuple): Employee = collections.namedtuple('Employee', ['name', 'id']) - The resulting class has an extra __annotations__ attribute, giving a - dict that maps field names to types. (The field names are also in + The types for each field name can be retrieved by calling + annotationlib.get_annotations(Employee). (The field names are also in the _fields attribute, which is part of the namedtuple API.) An alternative equivalent functional syntax is also accepted:: @@ -3182,7 +3164,7 @@ def __new__(cls, name, bases, ns, total=True, closed=None, This method is called when TypedDict is subclassed, or when TypedDict is instantiated. This way - TypedDict supports all three syntax forms described in its docstring. + TypedDict classes can be created through both class-based and functional syntax. Subclasses and instances of TypedDict return actual dictionaries. """ for base in bases: @@ -3207,9 +3189,9 @@ def __new__(cls, name, bases, ns, total=True, closed=None, if ns_annotations is not None: own_annotate = None own_annotations = ns_annotations - elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: - own_annotations = _lazy_annotationlib.call_annotate_function( - own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict + elif (own_annotate := annotationlib.get_annotate_from_class_namespace(ns)) is not None: + own_annotations = annotationlib.call_annotate_function( + own_annotate, annotationlib.Format.FORWARDREF, owner=tp_dict ) else: own_annotate = None @@ -3276,20 +3258,20 @@ def __annotate__(format): base_annotate = base.__annotate__ if base_annotate is None: continue - base_annos = _lazy_annotationlib.call_annotate_function( + base_annos = annotationlib.call_annotate_function( base_annotate, format, owner=base) annos.update(base_annos) if own_annotate is not None: - own = _lazy_annotationlib.call_annotate_function( + own = annotationlib.call_annotate_function( own_annotate, format, owner=tp_dict) - if format != _lazy_annotationlib.Format.STRING: + if format != annotationlib.Format.STRING: own = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own.items() } - elif format == _lazy_annotationlib.Format.STRING: - own = _lazy_annotationlib.annotations_to_string(own_annotations) - elif format in (_lazy_annotationlib.Format.FORWARDREF, _lazy_annotationlib.Format.VALUE): + elif format == annotationlib.Format.STRING: + own = annotationlib.annotations_to_string(own_annotations) + elif format in (annotationlib.Format.FORWARDREF, annotationlib.Format.VALUE): own = own_checked_annotations else: raise NotImplementedError(format) @@ -3336,14 +3318,22 @@ def TypedDict(typename, fields, /, *, total=True, closed=None, >>> Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') True - The type info can be accessed via the Point2D.__annotations__ dict, and - the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. + The type info can be accessed by calling annotationlib.get_annotations(Point2D), and + via the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. TypedDict supports an additional equivalent form:: Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality:: + to override this by using the NotRequired and Required special forms:: + + class Point2D(TypedDict): + x: int # the "x" key must always be present (Required is the default) + y: NotRequired[int] # the "y" key can be omitted + + This means that a Point2D TypedDict can have the "y" key omitted, but the "x" key must be present. + Items are required by default, so the Required special form is not necessary in this example. + In addition, the total argument to the TypedDict function can be used to make all items not required:: class Point2D(TypedDict, total=False): x: int @@ -3352,16 +3342,8 @@ class Point2D(TypedDict, total=False): This means that a Point2D TypedDict can have any of the keys omitted. A type checker is only expected to support a literal False or True as the value of the total argument. True is the default, and makes all items defined in the - class body be required. - - The Required and NotRequired special forms can also be used to mark - individual keys as being required or not required:: - - class Point2D(TypedDict): - x: int # the "x" key must always be present (Required is the default) - y: NotRequired[int] # the "y" key can be omitted - - See PEP 655 for more details on Required and NotRequired. + class body be required. The Required special form can be used to mark individual + keys as required in a total=False TypedDict. The ReadOnly special form can be used to mark individual keys as immutable for type checkers:: @@ -3395,7 +3377,7 @@ class Point3D(Point2D): by default, and it may not be used with the closed argument at the same time. - See PEP 728 for more information about closed and extra_items. + See PEPs 589, 655, 705, and 728 for more information. """ ns = {'__annotations__': dict(fields)} module = _caller() @@ -3425,7 +3407,7 @@ class Movie(TypedDict, total=False): year: int m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) @@ -3447,7 +3429,7 @@ class Movie(TypedDict): year: NotRequired[int] m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) """ @@ -3467,7 +3449,7 @@ class Movie(TypedDict): def mutate_movie(m: Movie) -> None: m["year"] = 1992 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. """ @@ -3554,8 +3536,8 @@ class IO(Generic[AnyStr]): classes (text vs. binary, read vs. write vs. read/write, append-only, unbuffered). The TextIO and BinaryIO subclasses below capture the distinctions between text vs. binary, which is - pervasive in the interface; however we currently do not offer a - way to track the other distinctions in the type system. + pervasive in the interface. For more precise types, define a custom + Protocol. """ __slots__ = () @@ -3645,7 +3627,7 @@ def __exit__(self, type, value, traceback, /) -> None: class BinaryIO(IO[bytes]): - """Typed version of the return of open() in binary mode.""" + """Typed approximation of the return of open() in binary mode.""" __slots__ = () @@ -3659,7 +3641,7 @@ def __enter__(self) -> BinaryIO: class TextIO(IO[str]): - """Typed version of the return of open() in text mode.""" + """Typed approximation of the return of open() in text mode.""" __slots__ = () @@ -3726,7 +3708,7 @@ def dataclass_transform( field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), **kwargs: Any, ) -> _IdentityCallable: - """Decorator to mark an object as providing dataclass-like behaviour. + """Decorator to mark an object as providing dataclass-like behavior. The decorator can be applied to a function, class, or metaclass. @@ -3886,7 +3868,7 @@ def __getattr__(attr): are only created on-demand here. """ if attr == "ForwardRef": - obj = _lazy_annotationlib.ForwardRef + obj = annotationlib.ForwardRef elif attr in {"Pattern", "Match"}: import re obj = _alias(getattr(re, attr), 1) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index a52950dad224ee7..697520246f0e3c6 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -1,5 +1,6 @@ """Loading unittests.""" +import inspect import os import re import sys @@ -84,8 +85,10 @@ def loadTestsFromTestCase(self, testCaseClass): raise TypeError("Test cases should not be derived from " "TestSuite. Maybe you meant to derive from " "TestCase?") - if testCaseClass in (case.TestCase, case.FunctionTestCase): - # We don't load any tests from base types that should not be loaded. + if (testCaseClass in (case.TestCase, case.FunctionTestCase) or + inspect.isabstract(testCaseClass)): + # We don't load any tests from base types that should not be loaded, + # and abstract base classes that can't be instantiated testCaseNames = [] else: testCaseNames = self.getTestCaseNames(testCaseClass) @@ -103,6 +106,7 @@ def loadTestsFromModule(self, module, *, pattern=None): isinstance(obj, type) and issubclass(obj, case.TestCase) and obj not in (case.TestCase, case.FunctionTestCase) + and not inspect.isabstract(obj) ): tests.append(self.loadTestsFromTestCase(obj)) @@ -181,6 +185,9 @@ def loadTestsFromName(self, name, module=None): elif (isinstance(obj, types.FunctionType) and isinstance(parent, type) and issubclass(parent, case.TestCase)): + if inspect.isabstract(parent): + raise TypeError( + "Cannot instantiate abstract test case %s" % parent.__name__) name = parts[-1] inst = parent(name) # static methods follow a different path diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index be99d93c78cca6c..6eeebf9657a3c7b 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -172,7 +172,7 @@ def _getParentArgParser(self): help='Show local variables in tracebacks') parser.add_argument('--durations', dest='durations', type=int, default=None, metavar="N", - help='Show the N slowest test cases (N=0 for all)') + help='Show the `N` slowest test cases (`N=0` for all)') if self.failfast is None: parser.add_argument('-f', '--failfast', dest='failfast', action='store_true', @@ -181,12 +181,12 @@ def _getParentArgParser(self): if self.catchbreak is None: parser.add_argument('-c', '--catch', dest='catchbreak', action='store_true', - help='Catch Ctrl-C and display results so far') + help='Catch `Ctrl-C` and display results so far') self.catchbreak = False if self.buffer is None: parser.add_argument('-b', '--buffer', dest='buffer', action='store_true', - help='Buffer stdout and stderr during tests') + help='Buffer `stdout` and `stderr` during tests') self.buffer = False if self.testNamePatterns is None: parser.add_argument('-k', dest='testNamePatterns', @@ -197,7 +197,7 @@ def _getParentArgParser(self): return parser def _getMainArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent], color=True) + parser = argparse.ArgumentParser(parents=[parent]) parser.prog = self.progName parser.print_help = self._print_help @@ -208,16 +208,16 @@ def _getMainArgParser(self, parent): return parser def _getDiscoveryArgParser(self, parent): - parser = argparse.ArgumentParser(parents=[parent], color=True) + parser = argparse.ArgumentParser(parents=[parent]) parser.prog = '%s discover' % self.progName parser.epilog = ('For test discovery all test modules must be ' 'importable from the top level directory of the ' 'project.') parser.add_argument('-s', '--start-directory', dest='start', - help="Directory to start discovery ('.' default)") + help="Directory to start discovery (`.` default)") parser.add_argument('-p', '--pattern', dest='pattern', - help="Pattern to match tests ('test*.py' default)") + help="Pattern to match tests (`test*.py` default)") parser.add_argument('-t', '--top-level-directory', dest='top', help='Top level directory of project (defaults to ' 'start directory)') diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 1cee67fa5d70946..49011faaa51eaed 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -24,7 +24,6 @@ ) -import asyncio import contextlib import io import inspect @@ -281,7 +280,6 @@ def reset_mock(): def _setup_async_mock(mock): - mock._is_coroutine = asyncio.coroutines._is_coroutine mock.await_count = 0 mock.await_args = None mock.await_args_list = _CallList() @@ -937,7 +935,7 @@ def _call_matcher(self, _call): return _call def assert_not_called(self): - """assert that the mock was never called. + """Assert that the mock was never called. """ if self.call_count != 0: msg = ("Expected '%s' to not have been called. Called %s times.%s" @@ -947,7 +945,7 @@ def assert_not_called(self): raise AssertionError(msg) def assert_called(self): - """assert that the mock was called at least once + """Assert that the mock was called at least once. """ if self.call_count == 0: msg = ("Expected '%s' to have been called." % @@ -955,7 +953,7 @@ def assert_called(self): raise AssertionError(msg) def assert_called_once(self): - """assert that the mock was called only once. + """Assert that the mock was called only once. """ if not self.call_count == 1: msg = ("Expected '%s' to have been called once. Called %s times.%s" @@ -965,7 +963,7 @@ def assert_called_once(self): raise AssertionError(msg) def assert_called_with(self, /, *args, **kwargs): - """assert that the last call was made with the specified arguments. + """Assert that the last call was made with the specified arguments. Raises an AssertionError if the args and keyword args passed in are different to the last call to the mock.""" @@ -987,7 +985,7 @@ def _error_message(): def assert_called_once_with(self, /, *args, **kwargs): - """assert that the mock was called exactly once and that call was + """Assert that the mock was called exactly once and that call was with the specified arguments.""" if not self.call_count == 1: msg = ("Expected '%s' to be called once. Called %s times.%s" @@ -999,7 +997,7 @@ def assert_called_once_with(self, /, *args, **kwargs): def assert_has_calls(self, calls, any_order=False): - """assert the mock has been called with the specified calls. + """Assert the mock has been called with the specified calls. The `mock_calls` list is checked for the calls. If `any_order` is False (the default) then the calls must be @@ -1044,7 +1042,7 @@ def assert_has_calls(self, calls, any_order=False): def assert_any_call(self, /, *args, **kwargs): - """assert the mock has been called with the specified arguments. + """Assert the mock has been called with the specified arguments. The assert passes if the mock has *ever* been called, unlike `assert_called_with` and `assert_called_once_with` that only pass if @@ -2287,13 +2285,6 @@ class AsyncMockMixin(Base): def __init__(self, /, *args, **kwargs): super().__init__(*args, **kwargs) - # iscoroutinefunction() checks _is_coroutine property to say if an - # object is a coroutine. Without this check it looks to see if it is a - # function/method, which in this case it is not (since it is an - # AsyncMock). - # It is set through __dict__ because when spec_set is True, this - # attribute is likely undefined. - self.__dict__['_is_coroutine'] = asyncio.coroutines._is_coroutine self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() @@ -3121,6 +3112,10 @@ def _mock_call(self, *args, **kwargs): return ret_value + def _increment_mock_call(self, /, *args, **kwargs): + with self._mock_calls_events_lock: + super()._increment_mock_call(*args, **kwargs) + def wait_until_called(self, *, timeout=_timeout_unset): """Wait until the mock object is called. diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 5f22d91aebd05f4..893fcba968c3ef8 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -4,11 +4,10 @@ import time import warnings -from _colorize import get_theme - from . import result from .case import _SubTest from .signals import registerResult +lazy from _colorize import get_theme __unittest = True diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index d64f678d235b6f9..82b95adbdc283ef 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -277,7 +277,7 @@ def _hostinfo(self): return hostname, port -_UNSPECIFIED = ['not specified'] +_UNSPECIFIED = sentinel("_UNSPECIFIED", repr="<not specified>") _MISSING_AS_NONE_DEFAULT = False class _ResultBase: diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 4009fd6b58f594f..e70eae800367840 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -7,7 +7,7 @@ 2) PSF license for Python 2.2 The robots.txt Exclusion Protocol is implemented as specified in - http://www.robotstxt.org/norobots-rfc.txt + RFC 9309 """ import collections @@ -21,19 +21,6 @@ RequestRate = collections.namedtuple("RequestRate", "requests seconds") -def normalize(path): - unquoted = urllib.parse.unquote(path, errors='surrogateescape') - return urllib.parse.quote(unquoted, errors='surrogateescape') - -def normalize_path(path): - path, sep, query = path.partition('?') - path = normalize(path) - if sep: - query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) - path += '?' + query - return path - - class RobotFileParser: """ This class provides a set of methods to read, parse and answer questions about a single robots.txt file. @@ -42,6 +29,7 @@ class RobotFileParser: def __init__(self, url=''): self.entries = [] + self.groups = {} self.sitemaps = [] self.default_entry = None self.disallow_all = False @@ -86,13 +74,13 @@ def read(self): self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) def _add_entry(self, entry): - if "*" in entry.useragents: - # the default entry is considered last - if self.default_entry is None: - # the first default entry wins - self.default_entry = entry - else: - self.entries.append(entry) + self.entries.append(entry) + for agent in entry.useragents: + agent = agent.lower() + if agent not in self.groups: + self.groups[agent] = entry + else: + self.groups[agent] = merge_entries(self.groups[agent], entry) def parse(self, lines): """Parse the input lines from a robots.txt file. @@ -100,6 +88,7 @@ def parse(self, lines): We allow that a user-agent: line is not preceded by one or more blank lines. """ + entries = [] # states: # 0: start state # 1: saw user-agent line @@ -109,14 +98,6 @@ def parse(self, lines): self.modified() for line in lines: - if not line: - if state == 1: - entry = Entry() - state = 0 - elif state == 2: - self._add_entry(entry) - entry = Entry() - state = 0 # remove optional comment and strip line i = line.find('#') if i >= 0: @@ -132,16 +113,23 @@ def parse(self, lines): if state == 2: self._add_entry(entry) entry = Entry() - entry.useragents.append(line[1]) + product_token = line[1] + entry.useragents.append(product_token) state = 1 elif line[0] == "disallow": if state != 0: - entry.rulelines.append(RuleLine(line[1], False)) state = 2 + try: + entry.rulelines.append(RuleLine(line[1], False)) + except ValueError: + pass elif line[0] == "allow": if state != 0: - entry.rulelines.append(RuleLine(line[1], True)) state = 2 + try: + entry.rulelines.append(RuleLine(line[1], True)) + except ValueError: + pass elif line[0] == "crawl-delay": if state != 0: # before trying to convert to int we need to make @@ -164,9 +152,18 @@ def parse(self, lines): # so it doesn't matter where you place it in your file." # Therefore we do not change the state of the parser. self.sitemaps.append(line[1]) - if state == 2: + if state != 0: self._add_entry(entry) + def _find_entry(self, useragent): + entry = self.groups.get(useragent.lower()) + if entry is not None: + return entry + for entry in self.groups.values(): + if entry.applies_to(useragent): + return entry + return self.groups.get('*') + def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" if self.disallow_all: @@ -179,43 +176,36 @@ def can_fetch(self, useragent, url): # calls can_fetch() before calling read(). if not self.last_checked: return False - # search for given user agent matches - # the first match counts # TODO: The private API is used in order to preserve an empty query. # This is temporary until the public API starts supporting this feature. parsed_url = urllib.parse._urlsplit(url, '') url = urllib.parse._urlunsplit(None, None, *parsed_url[2:]) - url = normalize_path(url) + url = normalize_uri(url) if not url: url = "/" - for entry in self.entries: - if entry.applies_to(useragent): - return entry.allowance(url) - # try the default entry last - if self.default_entry: - return self.default_entry.allowance(url) - # agent not found ==> access granted - return True + if url == '/robots.txt': + # The /robots.txt URI is implicitly allowed. + return True + entry = self._find_entry(useragent) + if entry is None: + return True + return entry.allowance(url) def crawl_delay(self, useragent): if not self.mtime(): return None - for entry in self.entries: - if entry.applies_to(useragent): - return entry.delay - if self.default_entry: - return self.default_entry.delay - return None + entry = self._find_entry(useragent) + if entry is None: + return None + return entry.delay def request_rate(self, useragent): if not self.mtime(): return None - for entry in self.entries: - if entry.applies_to(useragent): - return entry.req_rate - if self.default_entry: - return self.default_entry.req_rate - return None + entry = self._find_entry(useragent) + if entry is None: + return None + return entry.req_rate def site_maps(self): if not self.sitemaps: @@ -226,7 +216,7 @@ def __str__(self): entries = self.entries if self.default_entry is not None: entries = entries + [self.default_entry] - return '\n\n'.join(map(str, entries)) + return '\n\n'.join(filter(None, map(str, entries))) class RuleLine: """A rule line is a single "Allow:" (allowance==True) or "Disallow:" @@ -235,14 +225,42 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True - self.path = normalize_path(path) + path = re.sub(r'[*]{2,}', '*', path) + path = re.sub(r'[$][$*]+', '$', path) + path = normalize_pattern(path) + self.fullmatch = path.endswith('$') + path = path.rstrip('$') + if '$' in path: + raise ValueError('$ not at the end of path') + self.matcher = None + if '*' in path: + pattern = re.compile(translate_pattern(path), re.DOTALL) + if self.fullmatch: + self.matcher = pattern.fullmatch + else: + self.matcher = pattern.match + self.path = path self.allowance = allowance def applies_to(self, filename): - return self.path == "*" or filename.startswith(self.path) + # If the filename matches the rule, return the matching length plus 1. + # If it does not match, return 0. + if self.matcher is not None: + m = self.matcher(filename) + if m: + return m.end() + 1 + else: + if self.fullmatch: + if filename == self.path: + return len(self.path) + 1 + else: + if filename.startswith(self.path): + return len(self.path) + 1 + return 0 def __str__(self): - return ("Allow" if self.allowance else "Disallow") + ": " + self.path + return (("Allow" if self.allowance else "Disallow") + ": " + self.path + + ('$' if self.fullmatch else '')) class Entry: @@ -254,6 +272,8 @@ def __init__(self): self.req_rate = None def __str__(self): + if not self.useragents: + return '' ret = [] for agent in self.useragents: ret.append(f"User-agent: {agent}") @@ -262,27 +282,74 @@ def __str__(self): if self.req_rate is not None: rate = self.req_rate ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") - ret.extend(map(str, self.rulelines)) + if self.rulelines: + ret.extend(map(str, self.rulelines)) + else: + ret.append("Allow:") return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent""" + if useragent is None: + return '*' in self.useragents # split the name token and make it lower case useragent = useragent.split("/")[0].lower() for agent in self.useragents: - if agent == '*': - # we have the catch-all agent - return True - agent = agent.lower() - if agent in useragent: - return True + if agent != '*': + agent = agent.lower() + if agent in useragent: + return True return False def allowance(self, filename): """Preconditions: - our agent applies to this entry - - filename is URL encoded""" + - filename is URL encoded + """ + best_match = -1 + allowance = True for line in self.rulelines: - if line.applies_to(filename): - return line.allowance - return True + m = line.applies_to(filename) + if m: + if m > best_match: + best_match = m + allowance = line.allowance + elif m == best_match and not allowance: + allowance = line.allowance + return allowance + + +def normalize(path): + unquoted = urllib.parse.unquote(path, errors='surrogateescape') + return urllib.parse.quote(unquoted, errors='surrogateescape') + +def normalize_uri(path): + path, sep, query = path.partition('?') + path = normalize(path) + if sep: + query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + +def normalize_pattern(path): + path, sep, query = path.partition('?') + path = re.sub(r'[^*$]+', lambda m: normalize(m[0]), path) + if sep: + query = re.sub(r'[^=&*$]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + +def translate_pattern(path): + parts = list(map(re.escape, path.split('*'))) + for i in range(1, len(parts)-1): + parts[i] = f'(?>.*?{parts[i]})' + parts[-1] = f'.*{parts[-1]}' + return ''.join(parts) + +def merge_entries(e1, e2): + entry = Entry() + entry.useragents = list(filter(set(e2.useragents).__contains__, e1.useragents)) + entry.rulelines = e1.rulelines + e2.rulelines + entry.delay = e1.delay if e2.delay is None else e2.delay + entry.req_rate = e1.req_rate if e2.req_rate is None else e2.req_rate + return entry diff --git a/Lib/uuid.py b/Lib/uuid.py index 8c59581464b0d01..4bdcb67775a2ea1 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -955,7 +955,6 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Generate a UUID using the selected UUID function.", - color=True, ) parser.add_argument("-u", "--uuid", choices=uuid_funcs.keys(), @@ -963,14 +962,14 @@ def main(): help="function to generate the UUID") parser.add_argument("-n", "--namespace", metavar=f"{{any UUID,{','.join(namespaces)}}}", - help="uuid3/uuid5 only: " + help="`uuid3`/`uuid5` only: " "a UUID, or a well-known predefined UUID addressed " "by namespace name") parser.add_argument("-N", "--name", - help="uuid3/uuid5 only: " + help="`uuid3`/`uuid5` only: " "name used as part of generating the UUID") parser.add_argument("-C", "--count", metavar="NUM", type=int, default=1, - help="generate NUM fresh UUIDs") + help="generate `NUM` fresh UUIDs") args = parser.parse_args() uuid_func = uuid_funcs[args.uuid] diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 21f82125f5a7c40..bd2762d55ef6961 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -319,6 +319,14 @@ def setup_python(self, context): if not os.path.islink(path): os.chmod(path, 0o755) + if not self.symlinks and sys.platform == 'cygwin': + # Copy libpython DLL + libpython_dll = sysconfig.get_config_var('DLLLIBRARY') + if not os.path.exists(os.path.join(binpath, libpython_dll)): + exe_path = os.path.dirname(sys.executable) + shutil.copy(os.path.join(exe_path, libpython_dll), + os.path.join(binpath, libpython_dll)) + else: def setup_python(self, context): """ @@ -358,6 +366,9 @@ def setup_python(self, context): exe_t = f'3.{sys.version_info[1]}t' python_exe = os.path.join(dirname, f'python{exe_t}{exe_d}.exe') pythonw_exe = os.path.join(dirname, f'pythonw{exe_t}{exe_d}.exe') + if not os.path.isfile(python_exe): + python_exe = os.path.join(dirname, f'python{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_d}.exe') link_sources = { 'python.exe': python_exe, f'python{exe_d}.exe': python_exe, @@ -618,7 +629,6 @@ def main(args=None): 'activate it, e.g. by ' 'sourcing an activate script ' 'in its bin directory.', - color=True, ) parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', help='A directory to create the environment in.') diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 70673a265d41f80..657e4d481f6255a 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -17,11 +17,10 @@ deactivate () { # Call hash to forget past locations. Without forgetting # past locations the $PATH changes we made may not be respected. # See "man bash" for more details. hash is usually a builtin of your shell - hash -r 2> /dev/null + hash -r 2> /dev/null || true if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" - export PS1 unset _OLD_VIRTUAL_PS1 fi @@ -68,9 +67,8 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" PS1="("__VENV_PROMPT__") ${PS1:-}" - export PS1 fi # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -hash -r 2> /dev/null +hash -r 2> /dev/null || true diff --git a/Lib/venv/scripts/common/activate.fish b/Lib/venv/scripts/common/activate.fish index 284a7469c99b576..8d364ff88f8b828 100644 --- a/Lib/venv/scripts/common/activate.fish +++ b/Lib/venv/scripts/common/activate.fish @@ -15,10 +15,10 @@ function deactivate -d "Exit virtual environment and return to normal shell env if test -n "$_OLD_FISH_PROMPT_OVERRIDE" set -e _OLD_FISH_PROMPT_OVERRIDE # prevents error when using nested fish instances (Issue #93858) - if functions -q _old_fish_prompt - functions -e fish_prompt - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt + if builtin functions -q _old_fish_prompt + builtin functions -e fish_prompt + builtin functions -c _old_fish_prompt fish_prompt + builtin functions -e _old_fish_prompt end end @@ -26,7 +26,7 @@ function deactivate -d "Exit virtual environment and return to normal shell env set -e VIRTUAL_ENV_PROMPT if test "$argv[1]" != "nondestructive" # Self-destruct! - functions -e deactivate + builtin functions -e deactivate end end @@ -34,6 +34,9 @@ end deactivate nondestructive set -gx VIRTUAL_ENV __VENV_DIR__ +if string match -qr 'CYGWIN|MSYS|MINGW' (uname) + set -gx VIRTUAL_ENV (cygpath -u $VIRTUAL_ENV) +end set -gx _OLD_VIRTUAL_PATH $PATH set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH @@ -49,18 +52,21 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" # fish uses a function instead of an env var to generate the prompt. # Save the current fish_prompt function as the function _old_fish_prompt. - functions -c fish_prompt _old_fish_prompt + builtin functions -c fish_prompt _old_fish_prompt # With the original prompt function renamed, we can override with our own. + # Call every builtin through `builtin` so a user function that shadows + # `printf`, `set_color`, `echo`, or `source`/`.` cannot hijack the prompt + # (Issue #140006). function fish_prompt # Save the return status of the last command. set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s(%s)%s " (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal) + builtin printf "%s(%s)%s " (builtin set_color 4B8BBE) __VENV_PROMPT__ (builtin set_color normal) # Restore the return status of the previous command. - echo "exit $old_status" | . + builtin echo "exit $old_status" | builtin source - # Output the original/"old" prompt. _old_fish_prompt end diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 06f4753d73b7aed..4a3e791abb86bd2 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -13,8 +13,8 @@ @if defined _OLD_VIRTUAL_PROMPT @set PROMPT=%_OLD_VIRTUAL_PROMPT% @if defined _OLD_VIRTUAL_PYTHONHOME @set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -@set "_OLD_VIRTUAL_PROMPT=%PROMPT%" -@set "PROMPT=(__VENV_PROMPT__) %PROMPT%" +@if not defined VIRTUAL_ENV_DISABLE_PROMPT @set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +@if not defined VIRTUAL_ENV_DISABLE_PROMPT @set "PROMPT=(__VENV_PROMPT__) %PROMPT%" @if defined PYTHONHOME @set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% @set PYTHONHOME= diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 97aad6eea509eb5..ec8b544a6b05235 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -1,7 +1,8 @@ """Interfaces for launching and remotely controlling web browsers.""" -# Maintained by Georg Brandl. +import builtins # because we override open import os +lazy import plistlib import shlex import shutil import sys @@ -492,10 +493,15 @@ def register_standard_browsers(): _tryorder = [] if sys.platform == 'darwin': - register("MacOSX", None, MacOSXOSAScript('default')) - register("chrome", None, MacOSXOSAScript('google chrome')) - register("firefox", None, MacOSXOSAScript('firefox')) - register("safari", None, MacOSXOSAScript('safari')) + register("MacOS", None, MacOS('default')) + register("MacOSX", None, MacOS('default')) # backward compat alias + register("chrome", None, MacOS('google chrome')) + register("chromium", None, MacOS('chromium')) + register("firefox", None, MacOS('firefox')) + register("safari", None, MacOS('safari')) + register("opera", None, MacOS('opera')) + register("microsoft-edge", None, MacOS('microsoft edge')) + register("brave", None, MacOS('brave browser')) # macOS can use below Unix support (but we prefer using the macOS # specific stuff) @@ -614,8 +620,80 @@ def open(self, url, new=0, autoraise=True): # if sys.platform == 'darwin': + def _macos_default_browser_bundle_id(): + """Return the bundle ID of the default web browser. + + Reads the LaunchServices preferences file that macOS maintains + when the user sets a default browser. Returns 'com.apple.Safari' + if the file is absent or no https handler is recorded, because on + a fresh macOS installation Safari is the default browser and the + LaunchServices plist is not written until the user explicitly + changes their default browser. + """ + plist = os.path.expanduser( + '~/Library/Preferences/com.apple.LaunchServices/' + 'com.apple.launchservices.secure.plist' + ) + try: + with builtins.open(plist, 'rb') as f: + data = plistlib.load(f) + for handler in data.get('LSHandlers', []): + if handler.get('LSHandlerURLScheme') == 'https': + return (handler.get('LSHandlerRoleAll') + or handler.get('LSHandlerRoleViewer')) + except (OSError, KeyError, ValueError): + pass + return 'com.apple.Safari' + + class MacOS(BaseBrowser): + """Launcher class for macOS browsers, using /usr/bin/open. + + For http/https URLs with the default browser, /usr/bin/open is called + directly; macOS routes these to the registered browser. + + For all other URL schemes (e.g. file://) and for named browsers, + /usr/bin/open -b <bundle-id> is used so that the URL is always passed + to a browser application rather than dispatched by the OS file handler. + This prevents file injection attacks where a file:// URL pointing to an + executable bundle could otherwise be launched by the OS. + + Named browsers with known bundle IDs use -b; unknown names fall back + to -a. + """ + + _BUNDLE_IDS = { + 'google chrome': 'com.google.Chrome', + 'firefox': 'org.mozilla.firefox', + 'safari': 'com.apple.Safari', + 'chromium': 'org.chromium.Chromium', + 'opera': 'com.operasoftware.Opera', + 'microsoft edge': 'com.microsoft.edgemac', + 'brave browser': 'com.brave.Browser', + } + + def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) + self._check_url(url) + if self.name == 'default': + proto, sep, _ = url.partition(':') + if sep and proto.lower() in {'http', 'https'}: + cmd = ['/usr/bin/open', url] + else: + bundle_id = _macos_default_browser_bundle_id() + cmd = ['/usr/bin/open', '-b', bundle_id, url] + else: + bundle_id = self._BUNDLE_IDS.get(self.name.lower()) + if bundle_id: + cmd = ['/usr/bin/open', '-b', bundle_id, url] + else: + cmd = ['/usr/bin/open', '-a', self.name, url] + proc = subprocess.run(cmd, stderr=subprocess.DEVNULL) + return proc.returncode == 0 + class MacOSXOSAScript(BaseBrowser): def __init__(self, name='default'): + import warnings + warnings._deprecated("webbrowser.MacOSXOSAScript", remove=(3, 17)) super().__init__(name) def open(self, url, new=0, autoraise=True): @@ -734,7 +812,7 @@ def open(self, url, new=0, autoraise=True): def parse_args(arg_list: list[str] | None): import argparse parser = argparse.ArgumentParser( - description="Open URL in a web browser.", color=True, + description="Open URL in a web browser.", ) parser.add_argument("url", help="URL to open") diff --git a/Lib/xml/__init__.py b/Lib/xml/__init__.py index bf6d8ddfd04c934..ecfce1c6ae52cf2 100644 --- a/Lib/xml/__init__.py +++ b/Lib/xml/__init__.py @@ -16,5 +16,6 @@ """ +from .utils import * -__all__ = ["dom", "parsers", "sax", "etree"] +__all__ = ["dom", "parsers", "sax", "etree", "is_valid_name", "is_valid_text"] diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 85766e02b531ce2..53727d7940b3f2a 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -8,8 +8,8 @@ 2. Element represents a single node in this tree. Interactions with the whole document (reading and writing to/from files) are - usually done on the ElementTree level. Interactions with a single XML element - and its sub-elements are done on the Element level. + usually done on the ElementTree level. Interactions with a single XML + element and its sub-elements are done on the Element level. Element is a flexible container object designed to store hierarchical data structures in memory. It can be described as a cross between a list and a @@ -277,7 +277,8 @@ def find(self, path, namespaces=None): """Find first matching element by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -289,7 +290,8 @@ def findtext(self, path, default=None, namespaces=None): *path* is a string having either an element tag or an XPath, *default* is the value to return if the element was not found, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return text content of first matching element, or default value if none was found. Note that if an element is found having no text @@ -302,7 +304,8 @@ def findall(self, path, namespaces=None): """Find all matching subelements by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Returns list containing all matching elements in document order. @@ -313,7 +316,8 @@ def iterfind(self, path, namespaces=None): """Find all matching subelements by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return an iterable yielding all matching elements in document order. @@ -553,8 +557,8 @@ def _setroot(self, element): def parse(self, source, parser=None): """Load external XML document into element tree. - *source* is a file name or file object, *parser* is an optional parser - instance that defaults to XMLParser. + *source* is a file name or file object, *parser* is an optional + parser instance that defaults to XMLParser. ParseError is raised if the parser fails to parse the document. @@ -587,7 +591,8 @@ def parse(self, source, parser=None): def iter(self, tag=None): """Create and return tree iterator for the root element. - The iterator loops over all elements in this tree, in document order. + The iterator loops over all elements in this tree, in document + order. *tag* is a string with the tag name to iterate over (default is to return all elements). @@ -602,7 +607,8 @@ def find(self, path, namespaces=None): Same as getroot().find(path), which is Element.find() *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -624,7 +630,8 @@ def findtext(self, path, default=None, namespaces=None): Same as getroot().findtext(path), which is Element.findtext() *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -646,7 +653,8 @@ def findall(self, path, namespaces=None): Same as getroot().findall(path), which is Element.findall(). *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return list containing all matching elements in document order. @@ -693,24 +701,26 @@ def write(self, file_or_filename, """Write element tree to a file as XML. Arguments: - *file_or_filename* -- file name or a file object opened for writing + *file_or_filename* -- file name or a file object opened for + writing *encoding* -- the output encoding (default: US-ASCII) - *xml_declaration* -- bool indicating if an XML declaration should be - added to the output. If None, an XML declaration - is added if encoding IS NOT either of: - US-ASCII, UTF-8, or Unicode + *xml_declaration* -- bool indicating if an XML declaration should + be added to the output. If None, an XML + declaration is added if encoding IS NOT + either of: US-ASCII, UTF-8, or Unicode - *default_namespace* -- sets the default XML namespace (for "xmlns") + *default_namespace* -- sets the default XML namespace (for + "xmlns") *method* -- either "xml" (default), "html, "text", or "c14n" *short_empty_elements* -- controls the formatting of elements - that contain no content. If True (default) - they are emitted as a single self-closed - tag, otherwise they are emitted as a pair - of start/end tags + that contain no content. If True + (default) they are emitted as a single + self-closed tag, otherwise they are + emitted as a pair of start/end tags """ if self._root is None: @@ -907,17 +917,20 @@ def _serialize_xml(write, elem, qnames, namespaces, if elem.tail: write(_escape_cdata(elem.tail)) +_CDATA_CONTENT_ELEMENTS = {"script", "style", "xmp", "iframe", "noembed", + "noframes", "plaintext"} + HTML_EMPTY = {"area", "base", "basefont", "br", "col", "embed", "frame", "hr", "img", "input", "isindex", "link", "meta", "param", "source", - "track", "wbr"} + "track", "wbr", "plaintext"} def _serialize_html(write, elem, qnames, namespaces, **kwargs): tag = elem.tag text = elem.text if tag is Comment: - write("<!--%s-->" % _escape_cdata(text)) + write("<!--%s-->" % text) elif tag is ProcessingInstruction: - write("<?%s?>" % _escape_cdata(text)) + write("<?%s?>" % text) else: tag = qnames[tag] if tag is None: @@ -941,16 +954,19 @@ def _serialize_html(write, elem, qnames, namespaces, **kwargs): for k, v in items: if isinstance(k, QName): k = k.text - if isinstance(v, QName): - v = qnames[v.text] + k = qnames[k] + if v is None: + write(" %s" % k) # empty attr else: - v = _escape_attrib_html(v) - # FIXME: handle boolean attributes - write(" %s=\"%s\"" % (qnames[k], v)) + if isinstance(v, QName): + v = qnames[v.text] + else: + v = _escape_attrib_html(v) + write(" %s=\"%s\"" % (k, v)) write(">") ltag = tag.lower() if text: - if ltag == "script" or ltag == "style": + if ltag in _CDATA_CONTENT_ELEMENTS: write(text) else: write(_escape_cdata(text)) @@ -1083,9 +1099,9 @@ def tostring(element, encoding=None, method=None, *, is returned. Otherwise a bytestring is returned. *element* is an Element instance, *encoding* is an optional output - encoding defaulting to US-ASCII, *method* is an optional output which can - be one of "xml" (default), "html", "text" or "c14n", *default_namespace* - sets the default XML namespace (for "xmlns"). + encoding defaulting to US-ASCII, *method* is an optional output which + can be one of "xml" (default), "html", "text" or "c14n", + *default_namespace* sets the default XML namespace (for "xmlns"). Returns an (optionally) encoded string containing the XML data. @@ -1225,7 +1241,8 @@ def iterparse(source, events=None, parser=None): "end" events are reported. *source* is a filename or file object containing XML data, *events* is - a list of events to report back, *parser* is an optional parser instance. + a list of events to report back, *parser* is an optional parser + instance. Returns an iterator providing (event, elem) pairs. @@ -1761,10 +1778,11 @@ def flush(self): def canonicalize(xml_data=None, *, out=None, from_file=None, **options): """Convert XML to its C14N 2.0 serialised form. - If *out* is provided, it must be a file or file-like object that receives - the serialised canonical XML output (text, not bytes) through its ``.write()`` - method. To write to a file, open it in text mode with encoding "utf-8". - If *out* is not provided, this function returns the output as text string. + If *out* is provided, it must be a file or file-like object that + receives the serialised canonical XML output (text, not bytes) through + its ``.write()`` method. To write to a file, open it in text mode with + encoding "utf-8". If *out* is not provided, this function returns the + output as text string. Either *xml_data* (an XML string) or *from_file* (a file path or file-like object) must be provided as input. @@ -1798,19 +1816,22 @@ class C14NWriterTarget: Serialises parse events to XML C14N 2.0. The *write* function is used for writing out the resulting data stream - as text (not bytes). To write to a file, open it in text mode with encoding - "utf-8" and pass its ``.write`` method. + as text (not bytes). To write to a file, open it in text mode with + encoding "utf-8" and pass its ``.write`` method. Configuration options: - *with_comments*: set to true to include comments - - *strip_text*: set to true to strip whitespace before and after text content - - *rewrite_prefixes*: set to true to replace namespace prefixes by "n{number}" + - *strip_text*: set to true to strip whitespace before and after text + content + - *rewrite_prefixes*: set to true to replace namespace prefixes by + "n{number}" - *qname_aware_tags*: a set of qname aware tag names in which prefixes should be replaced in text content - - *qname_aware_attrs*: a set of qname aware attribute names in which prefixes - should be replaced in text content - - *exclude_attrs*: a set of attribute names that should not be serialised + - *qname_aware_attrs*: a set of qname aware attribute names in which + prefixes should be replaced in text content + - *exclude_attrs*: a set of attribute names that should not be + serialised - *exclude_tags*: a set of tag names that should not be serialised """ def __init__(self, write, *, diff --git a/Lib/xml/utils.py b/Lib/xml/utils.py new file mode 100644 index 000000000000000..532aa224dae6772 --- /dev/null +++ b/Lib/xml/utils.py @@ -0,0 +1,37 @@ +lazy import re as _re + + +def is_valid_name(name): + """Test whether a string is a valid element or attribute name.""" + # https://www.w3.org/TR/xml/#NT-Name + return _re.fullmatch( + # NameStartChar + '[' + ':A-Z_a-z' + '\xC0-\xD6\xD8-\xF6\xF8-\u02FF\u0370-\u037D\u037F-\u1FFF' + '\u200C\u200D' + '\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF' + '\uF900-\uFDCF\uFDF0-\uFFFD\U00010000-\U000EFFFF' + ']' + # NameChar + '[' + r'\-.0-9:A-Z_a-z' + '\xB7' + '\xC0-\xD6\xD8-\xF6\xF8-\u037D\u037F-\u1FFF' + '\u200C\u200D\u203F\u2040' + '\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF' + '\uF900-\uFDCF\uFDF0-\uFFFD\U00010000-\U000EFFFF' + ']*+', + name) is not None + +# https://www.w3.org/TR/xml/#charsets +_ILLEGAL_XML_CHAR = ( + '[' + '\x00-\x08\x0B\x0C\x0E-\x1F' # C0 controls except TAB, CR and LF + '\uD800-\uDFFF' # the surrogate blocks + '\uFFFE\uFFFF' # special Unicode characters + ']') + +def is_valid_text(data): + """Test whether a string is a sequence of legal XML 1.0 characters.""" + return _re.search(_ILLEGAL_XML_CHAR, data) is None diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index f441376d09c4aa2..84e4e4d11a7319e 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -965,7 +965,7 @@ def dumps(params, methodname=None, methodresponse=None, encoding=None, data = ( xmlheader, "<methodCall>\n" - "<methodName>", methodname, "</methodName>\n", + "<methodName>", escape(methodname), "</methodName>\n", data, "</methodCall>\n" ) diff --git a/Lib/zipapp.py b/Lib/zipapp.py index 7a4ef96ea0f0779..a1cef18ada9d05d 100644 --- a/Lib/zipapp.py +++ b/Lib/zipapp.py @@ -187,16 +187,16 @@ def main(args=None): """ import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('--output', '-o', default=None, help="The name of the output archive. " - "Required if SOURCE is an archive.") + "Required if `SOURCE` is an archive.") parser.add_argument('--python', '-p', default=None, help="The name of the Python interpreter to use " "(default: no shebang line).") parser.add_argument('--main', '-m', default=None, help="The main function of the application " - "(default: use an existing __main__.py).") + "(default: use an existing `__main__.py`).") parser.add_argument('--compress', '-c', action='store_true', help="Compress files with the deflate method. " "Files are stored uncompressed by default.") diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 1e0cc5f6234f281..418933a2e8d9e87 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -4,7 +4,6 @@ XXX references to utf-8 need further investigation. """ import binascii -import importlib.util import io import os import shutil @@ -419,6 +418,15 @@ def _sanitize_filename(filename): filename = filename.replace(os.altsep, "/") return filename +def _read_local_file_header(fp): + fheader = fp.read(sizeFileHeader) + if len(fheader) != sizeFileHeader: + raise BadZipFile("Truncated file header") + fheader = struct.unpack(structFileHeader, fheader) + if fheader[_FH_SIGNATURE] != stringFileHeader: + raise BadZipFile("Bad magic number for file header") + return fheader + class ZipInfo: """Class with attributes describing each file in the ZIP archive.""" @@ -566,8 +574,12 @@ def FileHeader(self, zip64=None): return header + filename + extra def _encodeFilenameFlags(self): + if self.flag_bits & _MASK_UTF_FILENAME: + encoding = 'ascii' + else: + encoding = 'cp437' try: - return self.filename.encode('ascii'), self.flag_bits + return self.filename.encode(encoding), self.flag_bits & ~_MASK_UTF_FILENAME except UnicodeEncodeError: return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME @@ -620,11 +632,12 @@ def _decodeExtra(self, filename_crc): def from_file(cls, filename, arcname=None, *, strict_timestamps=True): """Construct an appropriate ZipInfo for a file on the filesystem. - filename should be the path to a file or directory on the filesystem. + filename should be the path to a file or directory on the + filesystem. - arcname is the name which it will have within the archive (by default, - this will be the same as filename, but without a drive letter and with - leading path separators removed). + arcname is the name which it will have within the archive (by + default, this will be the same as filename, but without a drive + letter and with leading path separators removed). """ if isinstance(filename, os.PathLike): filename = os.fspath(filename) @@ -663,9 +676,12 @@ def _for_archive(self, archive): Return self. """ # gh-91279: Set the SOURCE_DATE_EPOCH to a specific timestamp - epoch = os.environ.get('SOURCE_DATE_EPOCH') - get_time = int(epoch) if epoch else time.time() - self.date_time = time.localtime(get_time)[:6] + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') + + if source_date_epoch: + self.date_time = time.gmtime(int(source_date_epoch))[:6] + else: + self.date_time = time.localtime(time.time())[:6] self.compress_type = archive.compression self.compress_level = archive.compresslevel @@ -778,6 +794,13 @@ def __init__(self): self._unconsumed = b'' self.eof = False + @property + def unused_data(self): + try: + return self._decomp.unused_data + except AttributeError: + return b'' + def decompress(self, data): if self._decomp is None: self._unconsumed += data @@ -1380,6 +1403,466 @@ def close(self): self._zipfile._writing = False +_REPACK_CHUNK_SIZE = 2**20 + + +class _ZipRepacker: + """Class for ZipFile repacking.""" + def __init__(self, *, strict_descriptor=True, + chunk_size=_REPACK_CHUNK_SIZE, debug=0): + self.debug = debug # Level of printing: 0 through 3 + self.chunk_size = chunk_size + self.strict_descriptor = strict_descriptor + + def _debug(self, level, *msg): + if self.debug >= level: + print(*msg) + + def repack(self, zfile, removed=None): + """ + Repack the ZIP file, stripping unreferenced local file entries. + + Assumes that local file entries (and the central directory, which is + mostly treated as the "last entry") are stored consecutively, with no + gaps or overlaps: + + 1. If any referenced entry overlaps with another, a `BadZipFile` error + is raised since safe repacking cannot be guaranteed. + + 2. Data before the first referenced entry is stripped only when it + appears to be a sequence of consecutive entries with no extra + following bytes; extra preceding bytes are preserved. + + 3. Data between referenced entries is stripped only when it appears to + be a sequence of consecutive entries with no extra preceding bytes; + extra following bytes are preserved. + + This is to prevent an unexpected data removal (false positive), though + a false negative may happen in certain rare cases. + + Examples: + + Stripping before the first referenced entry: + + [random bytes] + [unreferenced local file entry] + [random bytes] + <-- stripping start + [unreferenced local file entry] + [unreferenced local file entry] + <-- stripping end + [local file entry 1] (or central directory) + ... + + Stripping between referenced entries: + + ... + [local file entry] + <-- stripping start + [unreferenced local file entry] + [unreferenced local file entry] + <-- stripping end + [random bytes] + [unreferenced local file entry] + [random bytes] + [local file entry] (or central directory) + ... + + No stripping: + + [unreferenced local file entry] + [random bytes] + [local file entry 1] (or central directory) + ... + + No stripping: + + ... + [local file entry] + [random bytes] + [unreferenced local file entry] + [local file entry] (or central directory) + ... + + Side effects: + - Modifies the ZIP file in place. + - Updates zfile.start_dir to account for removed data. + - Sets zfile._didModify to True. + - Updates header_offset and clears _end_offset of referenced + ZipInfo instances. + + Parameters: + zfile: A ZipFile object representing the archive to repack. + removed: Optional. A sequence of ZipInfo instances representing + the previously removed entries. When provided, only their + corresponding local file entries are stripped. + """ + removed_zinfos = set(removed or ()) + + fp = zfile.fp + + # get a sorted filelist by header offset, in case the dir order + # doesn't match the actual entry order + filelist = (*zfile.filelist, *removed_zinfos) + filelist = sorted(filelist, key=lambda x: x.header_offset) + + # calculate each entry size and validate + entry_size_list = [] + used_entry_size_list = [] + for i, zinfo in enumerate(filelist): + try: + offset = filelist[i + 1].header_offset + except IndexError: + offset = zfile.start_dir + entry_size = offset - zinfo.header_offset + + # may raise on an invalid local file header + used_entry_size = self._calc_local_file_entry_size(fp, zinfo) + + self._debug(3, 'entry:', i, zinfo.orig_filename, + zinfo.header_offset, entry_size, used_entry_size) + if used_entry_size > entry_size: + raise BadZipFile( + f"Overlapped entries: {zinfo.orig_filename!r} ") + + if removed is not None and zinfo not in removed_zinfos: + used_entry_size = entry_size + + entry_size_list.append(entry_size) + used_entry_size_list.append(used_entry_size) + + # calculate the starting entry offset (bytes to skip) + if removed is None: + try: + offset = filelist[0].header_offset + except IndexError: + offset = zfile.start_dir + entry_offset = self._calc_initial_entry_offset(fp, offset) + else: + entry_offset = 0 + + # move file entries + for i, zinfo in enumerate(filelist): + entry_size = entry_size_list[i] + used_entry_size = used_entry_size_list[i] + + # update the header and move entry data to the new position + old_header_offset = zinfo.header_offset + zinfo.header_offset -= entry_offset + + if zinfo in removed_zinfos: + self._copy_bytes( + fp, + old_header_offset + used_entry_size, + zinfo.header_offset, + entry_size - used_entry_size, + ) + + # update entry_offset for subsequent files to follow + entry_offset += used_entry_size + + else: + if entry_offset > 0: + self._copy_bytes( + fp, + old_header_offset, + zinfo.header_offset, + used_entry_size, + ) + + stale_entry_size = self._validate_local_file_entry_sequence( + fp, + old_header_offset + used_entry_size, + old_header_offset + entry_size, + ) + + if stale_entry_size > 0: + self._copy_bytes( + fp, + old_header_offset + used_entry_size + stale_entry_size, + zinfo.header_offset + used_entry_size, + entry_size - used_entry_size - stale_entry_size, + ) + + # update entry_offset for subsequent files to follow + entry_offset += stale_entry_size + + # update state + zfile.start_dir -= entry_offset + zfile._didModify = True + + for zinfo in filelist: + zinfo._end_offset = None + + def _calc_initial_entry_offset(self, fp, data_offset): + checked_offsets = {} + if data_offset > 0: + self._debug(3, 'scanning file signatures before:', data_offset) + for pos in self._iter_scan_signature(fp, stringFileHeader, 0, data_offset): + self._debug(3, 'checking file signature at:', pos) + entry_size = self._validate_local_file_entry_sequence( + fp, pos, data_offset, checked_offsets) + if entry_size == data_offset - pos: + return entry_size + return 0 + + def _iter_scan_signature(self, fp, signature, start_offset, end_offset, + chunk_size=io.DEFAULT_BUFFER_SIZE): + sig_len = len(signature) + remainder = b'' + pos = start_offset + + while pos < end_offset: + # required for each loop since fp may be changed during each yield + fp.seek(pos) + + chunk = remainder + fp.read(min(chunk_size, end_offset - pos)) + + delta = pos - len(remainder) + idx = 0 + while True: + idx = chunk.find(signature, idx) + if idx == -1: + break + + yield delta + idx + idx += 1 + + remainder = chunk[-(sig_len - 1):] + pos += chunk_size + + def _validate_local_file_entry_sequence(self, fp, start_offset, end_offset, checked_offsets=None): + offset = start_offset + + while offset < end_offset: + self._debug(3, 'checking local file entry at:', offset) + + # Cache checked offsets to improve performance. + try: + entry_size = checked_offsets[offset] + except (KeyError, TypeError): + entry_size = self._validate_local_file_entry(fp, offset, end_offset) + if checked_offsets is not None: + checked_offsets[offset] = entry_size + else: + self._debug(3, 'read from checked cache:', offset) + + if entry_size is None: + break + + offset += entry_size + + return offset - start_offset + + def _validate_local_file_entry(self, fp, offset, end_offset): + fp.seek(offset) + try: + fheader = _read_local_file_header(fp) + except BadZipFile: + return None + + # Create a dummy ZipInfo to utilize parsing. + # Flush only the required information. + zinfo = ZipInfo() + zinfo.header_offset = offset + zinfo.flag_bits = fheader[_FH_GENERAL_PURPOSE_FLAG_BITS] + zinfo.compress_size = fheader[_FH_COMPRESSED_SIZE] + zinfo.file_size = fheader[_FH_UNCOMPRESSED_SIZE] + zinfo.CRC = fheader[_FH_CRC] + + filename = fp.read(fheader[_FH_FILENAME_LENGTH]) + zinfo.extra = fp.read(fheader[_FH_EXTRA_FIELD_LENGTH]) + pos = fp.tell() + + if pos > end_offset: + return None + + # parse zip64 + try: + zinfo._decodeExtra(crc32(filename)) + except BadZipFile: + return None + + dd_size = 0 + + if zinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR: + # According to the spec, these fields should be zero when data + # descriptor is used. Otherwise treat as a false positive on + # random bytes to return early, as scanning for data descriptor + # is rather expensive. + if not (zinfo.CRC == zinfo.compress_size == zinfo.file_size == 0): + return None + + zip64 = fheader[_FH_UNCOMPRESSED_SIZE] == 0xffffffff + + dd = self._scan_data_descriptor(fp, pos, end_offset, zip64) + if dd is None and not self.strict_descriptor: + if zinfo.flag_bits & _MASK_ENCRYPTED: + dd = False + else: + dd = self._scan_data_descriptor_no_sig_by_decompression( + fp, pos, end_offset, zip64, fheader[_FH_COMPRESSION_METHOD]) + if dd is False: + dd = self._scan_data_descriptor_no_sig(fp, pos, end_offset, zip64) + if dd is None: + return None + + zinfo.CRC, zinfo.compress_size, zinfo.file_size, dd_size = dd + + entry_size = ( + sizeFileHeader + + fheader[_FH_FILENAME_LENGTH] + fheader[_FH_EXTRA_FIELD_LENGTH] + + zinfo.compress_size + + dd_size + ) + + # Treat as a false positive if the entry would extend past end_offset, + # so callers never strip more bytes than the gap actually holds. + if offset + entry_size > end_offset: + return None + + return entry_size + + def _scan_data_descriptor(self, fp, offset, end_offset, zip64): + dd_fmt = '<LLQQ' if zip64 else '<LLLL' + dd_size = struct.calcsize(dd_fmt) + + # scan for signature and take the first valid descriptor + for pos in self._iter_scan_signature( + fp, struct.pack('<L', _DD_SIGNATURE), offset, end_offset + ): + fp.seek(pos) + dd = fp.read(min(dd_size, end_offset - pos)) + try: + _, crc, compress_size, file_size = struct.unpack(dd_fmt, dd) + except struct.error: + continue + + # @TODO: also check CRC to better guard from a false positive? + if pos - offset != compress_size: + continue + + return crc, compress_size, file_size, dd_size + + return None + + def _scan_data_descriptor_no_sig(self, fp, offset, end_offset, zip64, + chunk_size=io.DEFAULT_BUFFER_SIZE): + dd_fmt = '<LQQ' if zip64 else '<LLL' + dd_size = struct.calcsize(dd_fmt) + + pos = offset + remainder = b'' + + fp.seek(offset) + while pos < end_offset: + chunk = remainder + fp.read(min(chunk_size, end_offset - pos)) + + delta = pos - len(remainder) - offset + mv = memoryview(chunk) + for i in range(len(chunk) - dd_size + 1): + dd = mv[i:i + dd_size] + crc, compress_size, file_size = struct.unpack(dd_fmt, dd) + if delta + i != compress_size: + continue + + return crc, compress_size, file_size, dd_size + + remainder = chunk[-(dd_size - 1):] + pos += chunk_size + + return None + + def _scan_data_descriptor_no_sig_by_decompression(self, fp, offset, end_offset, zip64, method): + try: + decompressor = _get_decompressor(method) + except RuntimeError: + return False + + if decompressor is None: + return False + + dd_fmt = '<LQQ' if zip64 else '<LLL' + dd_size = struct.calcsize(dd_fmt) + + # early return and prevent potential `fp.read(-1)` + if end_offset - dd_size < offset: + return None + + try: + pos = self._trace_compressed_block_end(fp, offset, end_offset - dd_size, decompressor) + except Exception: + return None + + fp.seek(pos) + dd = fp.read(dd_size) + try: + crc, compress_size, file_size = struct.unpack(dd_fmt, dd) + except struct.error: + return None + if pos - offset != compress_size: + return None + + return crc, compress_size, file_size, dd_size + + def _trace_compressed_block_end(self, fp, offset, end_offset, decompressor, + chunk_size=io.DEFAULT_BUFFER_SIZE): + fp.seek(offset) + read_size = 0 + while True: + chunk = fp.read(min(chunk_size, end_offset - offset - read_size)) + if not chunk: + raise EOFError('Unexpected EOF while decompressing') + + # may raise on error + decompressor.decompress(chunk) + + read_size += len(chunk) + + if decompressor.eof: + unused_len = len(decompressor.unused_data) + return offset + read_size - unused_len + + def _calc_local_file_entry_size(self, fp, zinfo): + fp.seek(zinfo.header_offset) + fheader = _read_local_file_header(fp) + + if zinfo.flag_bits & _MASK_USE_DATA_DESCRIPTOR: + zip64 = fheader[_FH_UNCOMPRESSED_SIZE] == 0xffffffff + dd_fmt = '<LLQQ' if zip64 else '<LLLL' + fp.seek( + fheader[_FH_FILENAME_LENGTH] + fheader[_FH_EXTRA_FIELD_LENGTH] + + zinfo.compress_size, + os.SEEK_CUR, + ) + if fp.read(struct.calcsize('<L')) != struct.pack('<L', _DD_SIGNATURE): + dd_fmt = '<LQQ' if zip64 else '<LLL' + dd_size = struct.calcsize(dd_fmt) + else: + dd_size = 0 + + return ( + sizeFileHeader + + fheader[_FH_FILENAME_LENGTH] + fheader[_FH_EXTRA_FIELD_LENGTH] + + zinfo.compress_size + + dd_size + ) + + def _copy_bytes(self, fp, old_offset, new_offset, size): + read_size = 0 + while read_size < size: + fp.seek(old_offset + read_size) + data = fp.read(min(size - read_size, self.chunk_size)) + if not data: + raise BadZipFile( + "Truncated data while repacking " + f"(expected {size} bytes at offset {old_offset})" + ) + fp.seek(new_offset + read_size) + fp.write(data) + fp.flush() + read_size += len(data) + class ZipFile: """ Class with methods to open, read, write, close, list zip files. @@ -1392,19 +1875,19 @@ class ZipFile: mode: The mode can be either read 'r', write 'w', exclusive create 'x', or append 'a'. compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), - ZIP_BZIP2 (requires bz2), ZIP_LZMA (requires lzma), or - ZIP_ZSTANDARD (requires compression.zstd). - allowZip64: if True ZipFile will create files with ZIP64 extensions when - needed, otherwise it will raise an exception when this would - be necessary. - compresslevel: None (default for the given compression type) or an integer - specifying the level to pass to the compressor. - When using ZIP_STORED or ZIP_LZMA this keyword has no effect. - When using ZIP_DEFLATED integers 0 through 9 are accepted. - When using ZIP_BZIP2 integers 1 through 9 are accepted. - When using ZIP_ZSTANDARD integers -7 though 22 are common, - see the CompressionParameter enum in compression.zstd for - details. + ZIP_BZIP2 (requires bz2), ZIP_LZMA (requires lzma), or + ZIP_ZSTANDARD (requires compression.zstd). + allowZip64: if True ZipFile will create files with ZIP64 extensions + when needed, otherwise it will raise an exception when this + would be necessary. + compresslevel: None (default for the given compression type) or + an integer specifying the level to pass to the compressor. + When using ZIP_STORED or ZIP_LZMA this keyword has no effect. + When using ZIP_DEFLATED integers 0 through 9 are accepted. + When using ZIP_BZIP2 integers 1 through 9 are accepted. + When using ZIP_ZSTANDARD integers -7 though 22 are common, + see the CompressionParameter enum in compression.zstd for + details. """ @@ -1414,8 +1897,8 @@ class ZipFile: def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, compresslevel=None, *, strict_timestamps=True, metadata_encoding=None): - """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', - or append 'a'.""" + """Open the ZIP file with mode read 'r', write 'w', exclusive create + 'x', or append 'a'.""" if mode not in ('r', 'w', 'x', 'a'): raise ValueError("ZipFile requires mode 'r', 'w', 'x', or 'a'") @@ -1693,10 +2176,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): pwd is the password to decrypt files (only used for reading). - When writing, if the file size is not known in advance but may exceed - 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large - files. If the size is known in advance, it is best to pass a ZipInfo - instance for name, with zinfo.file_size set. + When writing, if the file size is not known in advance but may + exceed 2 GiB, pass force_zip64 to use the ZIP64 format, which can + handle large files. If the size is known in advance, it is best to + pass a ZipInfo instance for name, with zinfo.file_size set. """ if mode not in {"r", "w"}: raise ValueError('open() requires mode "r" or "w"') @@ -1732,12 +2215,7 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): self._fpclose, self._lock, lambda: self._writing) try: # Skip the file header: - fheader = zef_file.read(sizeFileHeader) - if len(fheader) != sizeFileHeader: - raise BadZipFile("Truncated file header") - fheader = struct.unpack(structFileHeader, fheader) - if fheader[_FH_SIGNATURE] != stringFileHeader: - raise BadZipFile("Bad magic number for file header") + fheader = _read_local_file_header(zef_file) fname = zef_file.read(fheader[_FH_FILENAME_LENGTH]) if fheader[_FH_EXTRA_FIELD_LENGTH]: @@ -1808,7 +2286,7 @@ def _open_to_write(self, zinfo, force_zip64=False): zinfo.compress_size = 0 zinfo.CRC = 0 - zinfo.flag_bits = 0x00 + zinfo.flag_bits = _MASK_UTF_FILENAME if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 @@ -1867,6 +2345,77 @@ def extractall(self, path=None, members=None, pwd=None): for zipinfo in members: self._extract_member(zipinfo, path, pwd) + def remove(self, zinfo_or_arcname): + """Remove a member from the archive.""" + if self.mode not in ('w', 'x', 'a'): + raise ValueError("remove() requires mode 'w', 'x', or 'a'") + if not self.fp: + raise ValueError( + "Attempt to write to ZIP archive that was already closed") + if self._writing: + raise ValueError( + "Can't write to ZIP archive while an open writing handle exists." + ) + + with self._lock: + # get the zinfo + if isinstance(zinfo_or_arcname, ZipInfo): + zinfo = zinfo_or_arcname + else: + # raise KeyError if arcname does not exist + zinfo = self.getinfo(zinfo_or_arcname) + + try: + self.filelist.remove(zinfo) + except ValueError: + raise KeyError('There is no item %r in the archive' % zinfo) from None + + try: + del self.NameToInfo[zinfo.filename] + except KeyError: + pass + + # Avoid missing entry if there is another entry having the same name, + # to prevent an error on `testzip()`. + # Reverse the order as NameToInfo normally stores the last added one. + for zi in reversed(self.filelist): + if zi.filename == zinfo.filename: + self.NameToInfo.setdefault(zi.filename, zi) + break + + self._didModify = True + + return zinfo + + def repack(self, removed=None, *, strict_descriptor=True, + chunk_size=_REPACK_CHUNK_SIZE): + """Repack a zip file, removing non-referenced file entries. + + The archive must be opened with mode 'a', as mode 'w'/'x' do not + truncate the file when closed. This cannot be simply changed as + they may be used on an unseekable file buffer, which disallows + truncation.""" + if self.mode != 'a': + raise ValueError("repack() requires mode 'a'") + if not self.fp: + raise ValueError( + "Attempt to write to ZIP archive that was already closed") + if self._writing: + raise ValueError( + "Can't write to ZIP archive while an open writing handle exists" + ) + + with self._lock: + self._writing = True + try: + repacker = _ZipRepacker( + strict_descriptor=strict_descriptor, + chunk_size=chunk_size, + ) + repacker.repack(self, removed) + finally: + self._writing = False + @classmethod def _sanitize_windows_name(cls, arcname, pathsep): """Replace bad characters and remove trailing dots from parts.""" @@ -2328,7 +2877,7 @@ def main(args=None): import argparse description = 'A simple command-line interface for zipfile module.' - parser = argparse.ArgumentParser(description=description, color=True) + parser = argparse.ArgumentParser(description=description) group = parser.add_mutually_exclusive_group(required=True) group.add_argument('-l', '--list', metavar='<zipfile>', help='Show listing of a zipfile') @@ -2341,7 +2890,7 @@ def main(args=None): group.add_argument('-t', '--test', metavar='<zipfile>', help='Test if a zipfile is valid') parser.add_argument('--metadata-encoding', metavar='<encoding>', - help='Specify encoding of member names for -l, -e and -t') + help='Specify encoding of member names for `-l`, `-e` and `-t`') args = parser.parse_args(args) encoding = args.metadata_encoding diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 98668c15d8bf94b..caa3a5b583bab32 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -26,7 +26,7 @@ def load_tzdata(key): # UnicodeEncodeError: If package_name or resource_name are not UTF-8, # such as keys containing a surrogate character. # IsADirectoryError: If package_name without a resource_name specified. - raise ZoneInfoNotFoundError(f"No time zone found with key {key}") + raise ZoneInfoNotFoundError(f"No time zone found with key {key!r}") def load_data(fobj): diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index c5f92a99a1e0766..394e42439f7710e 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.5.6", - url="https://github.com/openssl/openssl/releases/download/openssl-3.5.6/openssl-3.5.6.tar.gz", - checksum="deae7c80cba99c4b4f940ecadb3c3338b13cb77418409238e57d7f31f2a3b736", + name="OpenSSL 3.5.7", + url="https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz", + checksum="a8c0d28a529ca480f9f36cf5792e2cd21984552a3c8e4aa11a24aa31aeac98e8", buildrecipe=build_universal_openssl, configure=None, install=None, @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.50.4", - url="https://www.sqlite.org/2025/sqlite-autoconf-3500400.tar.gz", - checksum="a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18", + name="SQLite 3.53.2", + url="https://www.sqlite.org/2026/sqlite-autoconf-3530200.tar.gz", + checksum="588ad51949419a56ebe81fe56193d510c559eb94c9a57748387860b5d3069316", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -1158,7 +1158,6 @@ def buildPython(): print(" NOTE: --with-mimalloc=no pending resolution of weak linking issues") runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-mimalloc=no " - "--with-system-libmpdec " "--with-universal-archs=%s " "%s " "%s " diff --git a/Mac/IDLE/IDLE.app/Contents/Info.plist b/Mac/IDLE/IDLE.app/Contents/Info.plist index 8549e405e2a65a0..696625e64cdf326 100644 --- a/Mac/IDLE/IDLE.app/Contents/Info.plist +++ b/Mac/IDLE/IDLE.app/Contents/Info.plist @@ -1,9 +1,23 @@ <?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> - <key>CFBundleDevelopmentRegion</key> - <string>English</string> + <key>CFBundleName</key> + <string>IDLE</string> + <key>CFBundleIdentifier</key> + <string>org.python.IDLE</string> + <key>CFBundleVersion</key> + <string>%version%</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleSignature</key> + <string>????</string> + <key>CFBundleExecutable</key> + <string>IDLE</string> + <key>CFBundleIconFile</key> + <string>IDLE.icns</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> <key>CFBundleDocumentTypes</key> <array> <dict> @@ -34,26 +48,16 @@ <string>Editor</string> </dict> </array> - <key>CFBundleExecutable</key> - <string>IDLE</string> - <key>CFBundleGetInfoString</key> - <string>%version%, © 2001-2024 Python Software Foundation</string> - <key>CFBundleIconFile</key> - <string>IDLE.icns</string> - <key>CFBundleIdentifier</key> - <string>org.python.IDLE</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleName</key> - <string>IDLE</string> - <key>CFBundlePackageType</key> - <string>APPL</string> <key>CFBundleShortVersionString</key> <string>%version%</string> - <key>CFBundleSignature</key> - <string>????</string> - <key>CFBundleVersion</key> - <string>%version%</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2001 Python Software Foundation. All rights reserved.</string> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> <key>NSHighResolutionCapable</key> <true/> <key>CFBundleAllowMixedLocalizations</key> diff --git a/Mac/PythonLauncher/Info.plist.in b/Mac/PythonLauncher/Info.plist.in index ce8f27cd7d4de70..dd63187ab836b38 100644 --- a/Mac/PythonLauncher/Info.plist.in +++ b/Mac/PythonLauncher/Info.plist.in @@ -2,8 +2,22 @@ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> - <key>CFBundleDevelopmentRegion</key> - <string>en</string> + <key>CFBundleName</key> + <string>Python Launcher</string> + <key>CFBundleIdentifier</key> + <string>org.python.PythonLauncher</string> + <key>CFBundleVersion</key> + <string>%VERSION%</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleSignature</key> + <string>PytL</string> + <key>CFBundleExecutable</key> + <string>Python Launcher</string> + <key>CFBundleIconFile</key> + <string>PythonLauncher.icns</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> <key>CFBundleDocumentTypes</key> <array> <dict> @@ -37,28 +51,16 @@ <string>MyDocument</string> </dict> </array> - <key>CFBundleExecutable</key> - <string>Python Launcher</string> - <key>NSHumanReadableCopyright</key> - <string>Copyright © 2001 Python Software Foundation</string> - <key>CFBundleGetInfoString</key> - <string>%VERSION%, © 2001 Python Software Foundation</string> - <key>CFBundleIconFile</key> - <string>PythonLauncher.icns</string> - <key>CFBundleIdentifier</key> - <string>org.python.PythonLauncher</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleName</key> - <string>Python Launcher</string> - <key>CFBundlePackageType</key> - <string>APPL</string> <key>CFBundleShortVersionString</key> <string>%VERSION%</string> - <key>CFBundleSignature</key> - <string>PytL</string> - <key>CFBundleVersion</key> - <string>%VERSION%</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2001 Python Software Foundation. All rights reserved.</string> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> <key>NSMainNibFile</key> <string>MainMenu</string> <key>NSPrincipalClass</key> diff --git a/Mac/Resources/app/Info.plist.in b/Mac/Resources/app/Info.plist.in index a1fc1511c40e96d..07dc351398d7b2a 100644 --- a/Mac/Resources/app/Info.plist.in +++ b/Mac/Resources/app/Info.plist.in @@ -1,9 +1,23 @@ <?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> - <key>CFBundleDevelopmentRegion</key> - <string>English</string> + <key>CFBundleName</key> + <string>Python</string> + <key>CFBundleIdentifier</key> + <string>%bundleid%</string> + <key>CFBundleVersion</key> + <string>%version%</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleSignature</key> + <string>PytX</string> + <key>CFBundleExecutable</key> + <string>Python</string> + <key>CFBundleIconFile</key> + <string>PythonInterpreter.icns</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> <key>CFBundleDocumentTypes</key> <array> <dict> @@ -17,48 +31,33 @@ <string>Viewer</string> </dict> </array> - <key>CFBundleExecutable</key> - <string>Python</string> - <key>CFBundleGetInfoString</key> - <string>%version%, (c) 2001-2024 Python Software Foundation.</string> + <key>CFBundleShortVersionString</key> + <string>%version%</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2001 Python Software Foundation. All rights reserved.</string> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> + <key>NSHighResolutionCapable</key> + <true/> + <key>NSAppleScriptEnabled</key> + <true/> + <key>CFBundleAllowMixedLocalizations</key> + <true/> <key>CFBundleHelpBookFolder</key> <array> <string>Documentation</string> - <string>PythonDocumentation</string> </array> <key>CFBundleHelpBookName</key> - <string>MacPython Help</string> + <string>Python Help</string> <key>CFBundleHelpTOCFile</key> <string>index.html</string> - <key>CFBundleIconFile</key> - <string>PythonInterpreter.icns</string> - <key>CFBundleIdentifier</key> - <string>%bundleid%</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleLongVersionString</key> - <string>%version%, (c) 2001-2024 Python Software Foundation.</string> - <key>CFBundleName</key> - <string>Python</string> - <key>CFBundlePackageType</key> - <string>APPL</string> - <key>CFBundleShortVersionString</key> - <string>%version%</string> - <key>CFBundleSignature</key> - <string>PytX</string> - <key>CFBundleVersion</key> - <string>%version%</string> <key>CSResourcesFileMapped</key> <true/> <key>LSRequiresCarbon</key> <true/> - <key>NSAppleScriptEnabled</key> - <true/> - <key>NSHumanReadableCopyright</key> - <string>(c) 2001-2024 Python Software Foundation.</string> - <key>NSHighResolutionCapable</key> - <true/> - <key>CFBundleAllowMixedLocalizations</key> - <true/> </dict> </plist> diff --git a/Mac/Resources/framework/Info.plist.in b/Mac/Resources/framework/Info.plist.in index 4c42971ed90ee4c..41400c91a5f1b0e 100644 --- a/Mac/Resources/framework/Info.plist.in +++ b/Mac/Resources/framework/Info.plist.in @@ -1,29 +1,31 @@ <?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist SYSTEM "file://localhost/System/Library/DTDs/PropertyList.dtd"> -<plist version="0.9"> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> <dict> - <key>CFBundleDevelopmentRegion</key> - <string>English</string> - <key>CFBundleExecutable</key> + <key>CFBundleName</key> <string>Python</string> - <key>CFBundleGetInfoString</key> - <string>Python Runtime and Library</string> <key>CFBundleIdentifier</key> <string>@PYTHONFRAMEWORKIDENTIFIER@</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleName</key> - <string>Python</string> + <key>CFBundleVersion</key> + <string>%VERSION%</string> <key>CFBundlePackageType</key> <string>FMWK</string> - <key>CFBundleShortVersionString</key> - <string>%VERSION%, (c) 2001-2024 Python Software Foundation.</string> - <key>CFBundleLongVersionString</key> - <string>%VERSION%, (c) 2001-2024 Python Software Foundation.</string> <key>CFBundleSignature</key> <string>????</string> - <key>CFBundleVersion</key> + <key>CFBundleExecutable</key> + <string>Python</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundleShortVersionString</key> <string>%VERSION%</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2001 Python Software Foundation. All rights reserved.</string> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> <key>CFBundleAllowMixedLocalizations</key> <true/> </dict> diff --git a/Makefile.pre.in b/Makefile.pre.in index 5789d33e7f4456f..b9914369ad1bedc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -215,6 +215,8 @@ MACOSX_DEPLOYMENT_TARGET=@CONFIGURE_MACOSX_DEPLOYMENT_TARGET@ # the build, and is only listed here so it will be included in sysconfigdata. IPHONEOS_DEPLOYMENT_TARGET=@IPHONEOS_DEPLOYMENT_TARGET@ +BUILD_DETAILS=@BUILD_DETAILS@ + # Option to install to strip binaries STRIPFLAG=-s @@ -231,7 +233,6 @@ RUNSHARED= @RUNSHARED@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries -LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a # HACL* build configuration @@ -470,6 +471,7 @@ PYTHON_OBJS= \ Python/instruction_sequence.o \ Python/intrinsics.o \ Python/jit.o \ + Python/jit_publish.o \ $(JIT_OBJS) \ Python/legacy_tracing.o \ Python/lock.o \ @@ -497,6 +499,8 @@ PYTHON_OBJS= \ Python/qsbr.o \ Python/bootstrap_hash.o \ Python/specialize.o \ + Python/slots.o \ + Python/slots_generated.o \ Python/stackrefs.o \ Python/structmember.o \ Python/symtable.o \ @@ -603,45 +607,6 @@ LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@ DTRACE_DEPS = \ Python/ceval.o Python/gc.o Python/import.o Python/sysmodule.o -########################################################################## -# decimal's libmpdec - -LIBMPDEC_OBJS= \ - Modules/_decimal/libmpdec/basearith.o \ - Modules/_decimal/libmpdec/constants.o \ - Modules/_decimal/libmpdec/context.o \ - Modules/_decimal/libmpdec/convolute.o \ - Modules/_decimal/libmpdec/crt.o \ - Modules/_decimal/libmpdec/difradix2.o \ - Modules/_decimal/libmpdec/fnt.o \ - Modules/_decimal/libmpdec/fourstep.o \ - Modules/_decimal/libmpdec/io.o \ - Modules/_decimal/libmpdec/mpalloc.o \ - Modules/_decimal/libmpdec/mpdecimal.o \ - Modules/_decimal/libmpdec/numbertheory.o \ - Modules/_decimal/libmpdec/sixstep.o \ - Modules/_decimal/libmpdec/transpose.o - # _decimal does not use signaling API - # Modules/_decimal/libmpdec/mpsignal.o - -LIBMPDEC_HEADERS= \ - $(srcdir)/Modules/_decimal/libmpdec/basearith.h \ - $(srcdir)/Modules/_decimal/libmpdec/bits.h \ - $(srcdir)/Modules/_decimal/libmpdec/constants.h \ - $(srcdir)/Modules/_decimal/libmpdec/convolute.h \ - $(srcdir)/Modules/_decimal/libmpdec/crt.h \ - $(srcdir)/Modules/_decimal/libmpdec/difradix2.h \ - $(srcdir)/Modules/_decimal/libmpdec/fnt.h \ - $(srcdir)/Modules/_decimal/libmpdec/fourstep.h \ - $(srcdir)/Modules/_decimal/libmpdec/io.h \ - $(srcdir)/Modules/_decimal/libmpdec/mpalloc.h \ - $(srcdir)/Modules/_decimal/libmpdec/mpdecimal.h \ - $(srcdir)/Modules/_decimal/libmpdec/numbertheory.h \ - $(srcdir)/Modules/_decimal/libmpdec/sixstep.h \ - $(srcdir)/Modules/_decimal/libmpdec/transpose.h \ - $(srcdir)/Modules/_decimal/libmpdec/typearith.h \ - $(srcdir)/Modules/_decimal/libmpdec/umodarith.h - ########################################################################## # pyexpat's expat library @@ -811,11 +776,11 @@ list-targets: .PHONY: build_all build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sharedmods \ - gdbhooks Programs/_testembed scripts checksharedmods rundsymutil build-details.json + gdbhooks Programs/_testembed scripts checksharedmods rundsymutil $(BUILD_DETAILS) .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ - python-config checksharedmods build-details.json + python-config checksharedmods $(BUILD_DETAILS) .PHONY: build_emscripten build_emscripten: build_wasm web_example web_example_pyrepl_jspi @@ -832,7 +797,7 @@ check-clean-src: echo "Building Python out of the source tree (in $(abs_builddir)) requires a clean source tree ($(abs_srcdir))" ; \ echo "Build artifacts such as .o files, executables, and Python/frozen_modules/*.h must not exist within $(srcdir)." ; \ echo "Try to run:" ; \ - echo " (cd \"$(srcdir)\" && make clean || git clean -fdx -e Doc/venv)" ; \ + echo " (cd \"$(srcdir)\" && make distclean || git clean -fdx -e Doc/venv)" ; \ exit 1; \ fi @@ -959,7 +924,6 @@ coverage-lcov: @lcov $(COVERAGE_LCOV_OPTIONS) --remove $(COVERAGE_INFO) \ '*/Modules/_hacl/*' \ '*/Modules/_ctypes/libffi*/*' \ - '*/Modules/_decimal/libmpdec/*' \ '*/Modules/expat/*' \ '*/Modules/xx*.c' \ '*/Python/pyfpe.c' \ @@ -1017,8 +981,8 @@ pybuilddir.txt: $(PYTHON_FOR_BUILD_DEPS) exit 1 ; \ fi -build-details.json: pybuilddir.txt - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/build-details.json +$(BUILD_DETAILS): pybuilddir.txt + $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/generate-build-details.py `cat pybuilddir.txt`/$(BUILD_DETAILS) # Build static library $(LIBRARY): $(LIBRARY_OBJS) @@ -1244,12 +1208,13 @@ PYTHON_HEADERS= \ $(srcdir)/Include/refcount.h \ $(srcdir)/Include/setobject.h \ $(srcdir)/Include/sliceobject.h \ + $(srcdir)/Include/slots.h \ + $(srcdir)/Include/slots_generated.h \ $(srcdir)/Include/structmember.h \ $(srcdir)/Include/structseq.h \ $(srcdir)/Include/sysmodule.h \ $(srcdir)/Include/traceback.h \ $(srcdir)/Include/tupleobject.h \ - $(srcdir)/Include/typeslots.h \ $(srcdir)/Include/unicodeobject.h \ $(srcdir)/Include/warnings.h \ $(srcdir)/Include/weakrefobject.h \ @@ -1432,6 +1397,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_setobject.h \ $(srcdir)/Include/internal/pycore_signal.h \ $(srcdir)/Include/internal/pycore_sliceobject.h \ + $(srcdir)/Include/internal/pycore_slots.h \ + $(srcdir)/Include/internal/pycore_slots_generated.h \ $(srcdir)/Include/internal/pycore_stats.h \ $(srcdir)/Include/internal/pycore_strhex.h \ $(srcdir)/Include/internal/pycore_stackref.h \ @@ -1465,60 +1432,6 @@ PYTHON_HEADERS= \ \ $(srcdir)/Python/stdlib_module_names.h -########################################################################## -# Build static libmpdec.a -LIBMPDEC_CFLAGS=@LIBMPDEC_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) - -# "%.o: %c" is not portable -Modules/_decimal/libmpdec/basearith.o: $(srcdir)/Modules/_decimal/libmpdec/basearith.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/basearith.c - -Modules/_decimal/libmpdec/constants.o: $(srcdir)/Modules/_decimal/libmpdec/constants.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/constants.c - -Modules/_decimal/libmpdec/context.o: $(srcdir)/Modules/_decimal/libmpdec/context.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/context.c - -Modules/_decimal/libmpdec/convolute.o: $(srcdir)/Modules/_decimal/libmpdec/convolute.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/convolute.c - -Modules/_decimal/libmpdec/crt.o: $(srcdir)/Modules/_decimal/libmpdec/crt.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/crt.c - -Modules/_decimal/libmpdec/difradix2.o: $(srcdir)/Modules/_decimal/libmpdec/difradix2.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/difradix2.c - -Modules/_decimal/libmpdec/fnt.o: $(srcdir)/Modules/_decimal/libmpdec/fnt.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/fnt.c - -Modules/_decimal/libmpdec/fourstep.o: $(srcdir)/Modules/_decimal/libmpdec/fourstep.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/fourstep.c - -Modules/_decimal/libmpdec/io.o: $(srcdir)/Modules/_decimal/libmpdec/io.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/io.c - -Modules/_decimal/libmpdec/mpalloc.o: $(srcdir)/Modules/_decimal/libmpdec/mpalloc.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/mpalloc.c - -Modules/_decimal/libmpdec/mpdecimal.o: $(srcdir)/Modules/_decimal/libmpdec/mpdecimal.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/mpdecimal.c - -Modules/_decimal/libmpdec/mpsignal.o: $(srcdir)/Modules/_decimal/libmpdec/mpsignal.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/mpsignal.c - -Modules/_decimal/libmpdec/numbertheory.o: $(srcdir)/Modules/_decimal/libmpdec/numbertheory.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/numbertheory.c - -Modules/_decimal/libmpdec/sixstep.o: $(srcdir)/Modules/_decimal/libmpdec/sixstep.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/sixstep.c - -Modules/_decimal/libmpdec/transpose.o: $(srcdir)/Modules/_decimal/libmpdec/transpose.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS) - $(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/transpose.c - -$(LIBMPDEC_A): $(LIBMPDEC_OBJS) - -rm -f $@ - $(AR) $(ARFLAGS) $@ $(LIBMPDEC_OBJS) - ########################################################################## # Build static libexpat.a LIBEXPAT_CFLAGS=@LIBEXPAT_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) @@ -1703,6 +1616,8 @@ Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS) + # Dummy pybuilddir.txt is needed for _bootstrap_python to be runnable + @echo "none" > ./pybuilddir.txt ############################################################################ @@ -1941,7 +1856,7 @@ regen-unicodedata: # "clinic" is regenerated implicitly via "regen-global-objects". .PHONY: regen-all -regen-all: regen-cases regen-typeslots \ +regen-all: regen-cases regen-slots \ regen-token regen-ast regen-keyword regen-sre regen-frozen \ regen-pegen-metaparser regen-pegen regen-test-frozenmain \ regen-test-levenshtein regen-global-objects @@ -2314,16 +2229,17 @@ Python/import.o: $(srcdir)/Include/pydtrace.h Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) -Objects/typeobject.o: Objects/typeslots.inc - .PHONY: regen-typeslots regen-typeslots: - # Regenerate Objects/typeslots.inc from Include/typeslotsh - # using Objects/typeslots.py - $(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \ - < $(srcdir)/Include/typeslots.h \ - $(srcdir)/Objects/typeslots.inc.new - $(UPDATE_FILE) $(srcdir)/Objects/typeslots.inc $(srcdir)/Objects/typeslots.inc.new + echo 'NOTE: "regen-typeslots" was renamed to "regen-slots"' + $(MAKE) regen-slots + +.PHONY: regen-slots +regen-slots: Python/slots.toml + # Regenerate {Python,Include}/slots_generated.{c,h} + # from Python/slots.toml using Tools/build/generate_slots.py + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_slots.py \ + --generate-all $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) @@ -2437,7 +2353,7 @@ multissltest: all # Only the main install gets a build-details.json. .PHONY: install install: @FRAMEWORKINSTALLFIRST@ @INSTALLTARGETS@ @FRAMEWORKINSTALLLAST@ - $(INSTALL_DATA) `cat pybuilddir.txt`/build-details.json $(DESTDIR)$(LIBDEST); \ + $(INSTALL_DATA) `cat pybuilddir.txt`/$(BUILD_DETAILS) $(DESTDIR)$(LIBDEST); \ if test "x$(ENSUREPIP)" != "xno" ; then \ case $(ENSUREPIP) in \ upgrade) ensurepip="--upgrade" ;; \ @@ -2759,6 +2675,20 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_lazy_import/data \ test/test_lazy_import/data/pkg \ test/test_lazy_import/data/badsyntax \ + test/test_lazy_import/data/circular_import_pkg \ + test/test_lazy_import/data/metasyntactic \ + test/test_lazy_import/data/metasyntactic/foo \ + test/test_lazy_import/data/metasyntactic/foo/ack \ + test/test_lazy_import/data/metasyntactic/foo/bar \ + test/test_lazy_import/data/metasyntactic/foo/bar/baz \ + test/test_lazy_import/data/metasyntactic/foo/bar/baz/qux \ + test/test_lazy_import/data/metasyntactic/foo/bar/thud \ + test/test_lazy_import/data/metasyntactic/plugh \ + test/test_lazy_import/data/metasyntactic/waldo \ + test/test_lazy_import/data/metasyntactic/waldo/fred \ + test/test_lazy_import/data/module_same_name_var_order1 \ + test/test_lazy_import/data/module_same_name_var_order2 \ + test/test_lazy_import/data/versioned \ test/test_module \ test/test_multiprocessing_fork \ test/test_multiprocessing_forkserver \ @@ -3199,12 +3129,33 @@ config.status: $(srcdir)/configure .PRECIOUS: config.status $(BUILDPYTHON) Makefile Makefile.pre -Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S +Python/asm_trampoline_x86_64.o: $(srcdir)/Python/asm_trampoline_x86_64.S + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +Python/asm_trampoline_aarch64.o: $(srcdir)/Python/asm_trampoline_aarch64.S $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< +Python/asm_trampoline_riscv64.o: $(srcdir)/Python/asm_trampoline_riscv64.S + $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< + +# On macOS universal2 builds, $(PY_CORE_CFLAGS) contains "-arch arm64 -arch x86_64", +# which would produce fat .o files containing both architectures for each .S input. +# lipo -create then refuses to combine them because they share architectures. +# Build each per-arch object with a single -arch flag before merging with lipo. +Python/asm_trampoline_universal2.o: $(srcdir)/Python/asm_trampoline_aarch64.S $(srcdir)/Python/asm_trampoline_x86_64.S + $(CC) -c $(filter-out -arch arm64 x86_64,$(PY_CORE_CFLAGS)) -arch arm64 \ + -o Python/asm_trampoline_arm64-apple-darwin.o $(srcdir)/Python/asm_trampoline_aarch64.S + $(CC) -c $(filter-out -arch arm64 x86_64,$(PY_CORE_CFLAGS)) -arch x86_64 \ + -o Python/asm_trampoline_x86_64-apple-darwin.o $(srcdir)/Python/asm_trampoline_x86_64.S + lipo -create -output $@ \ + Python/asm_trampoline_arm64-apple-darwin.o \ + Python/asm_trampoline_x86_64-apple-darwin.o + rm -f Python/asm_trampoline_arm64-apple-darwin.o \ + Python/asm_trampoline_x86_64-apple-darwin.o + Python/emscripten_trampoline_inner.wasm: $(srcdir)/Python/emscripten_trampoline_inner.c # emcc has a path that ends with emsdk/upstream/emscripten/emcc, we're looking for emsdk/upstream/bin/clang. - $$(dirname $$(dirname $(CC)))/bin/clang -o $@ $< -mgc -O2 -Wl,--no-entry -Wl,--import-table -Wl,--import-memory -target wasm32-unknown-unknown -nostdlib + $$(em-config LLVM_ROOT)/clang -o $@ $< -mgc -O2 -Wl,--no-entry -Wl,--import-table -Wl,--import-memory -target wasm32-unknown-unknown -nostdlib Python/emscripten_trampoline_wasm.c: Python/emscripten_trampoline_inner.wasm $(PYTHON_FOR_REGEN) $(srcdir)/Platforms/emscripten/prepare_external_wasm.py $< $@ getWasmTrampolineModule @@ -3323,7 +3274,9 @@ docclean: # data. The PGO data is only valid if source code remains unchanged. .PHONY: clean-retain-profile clean-retain-profile: pycremoval - find . -name '*.[oa]' -exec rm -f {} ';' + # Keep the generated JIT shim objects with the rest of the JIT generated + # files: they are regenerated as a group and tracked by .jit-stamp. + find . -name '*.[oa]' ! -name 'jit_shim*.o' -exec rm -f {} ';' find . -name '*.s[ol]' -exec rm -f {} ';' find . -name '*.so.[0-9]*.[0-9]*' -exec rm -f {} ';' find . -name '*.lto' -exec rm -f {} ';' @@ -3492,7 +3445,6 @@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/uni MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ -MODULE__DECIMAL_DEPS=@LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h diff --git a/Misc/ACKS b/Misc/ACKS index 234d0d2d0a2a164..1f2049da56c2da7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -144,6 +144,7 @@ Bas van Beek Ian Beer Stefan Behnel Reimer Behrends +Tomi Belan Maxime Bélanger Ben Bell Thomas Bellman @@ -1225,6 +1226,7 @@ Owen Martin Sidney San Martín Westley Martínez Sébastien Martini +Iván Márton Roger Masse Nick Mathewson Simon Mathieu @@ -1761,6 +1763,7 @@ Charlie Shepherd Bruce Sherwood Gregory Shevchenko Hai Shi +Daniel Shields Alexander Shigin Pete Shinners Michael Shiplett @@ -2142,6 +2145,7 @@ Xiang Zhang Robert Xiao Florent Xicluna Yanbo, Xie +Ivy Xu Kaisheng Xu Xinhang Xu Arnon Yaari diff --git a/Misc/Brewfile b/Misc/Brewfile new file mode 100644 index 000000000000000..c799f099957f757 --- /dev/null +++ b/Misc/Brewfile @@ -0,0 +1,14 @@ +brew "gdbm" +brew "mpdecimal" +brew "openssl@3.5" +brew "pkg-config" +brew "tcl-tk@9" +brew "xz" +brew "zstd" + +brew "bzip2" if OS.linux? +brew "libedit" if OS.linux? +brew "libffi" if OS.linux? +brew "ncurses" if OS.linux? +brew "unzip" if OS.linux? +brew "zlib-ng-compat" if OS.linux? diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index cd419dfaaee2e8d..16eca7a55db746e 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -622,7 +622,7 @@ Harmonized :func:`random.randrange` argument handling to match :func:`range`. .. nonce: O4VcCY .. section: Library -Restore compatibility for ``uname_result`` around deepcopy and _replace. +Restore compatibility for :class:`os.uname_result` around deepcopy and _replace. .. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index 9c73ae95d10a4c6..a4792987ab7e2c6 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -957,7 +957,7 @@ Fix ``make regen-test-levenshtein`` for out-of-tree builds. .. nonce: eVXGEx .. section: Build -Don't use vendored ``libmpdec`` headers if :option:`--with-system-libmpdec` +Don't use vendored ``libmpdec`` headers if :option:`!--with-system-libmpdec` is passed to :program:`configure`. Don't use vendored ``libexpat`` headers if :option:`--with-system-expat` is passed to :program:`configure`. diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst index d2c717afcb6e8d8..c71a66757c65662 100644 --- a/Misc/NEWS.d/3.12.0a3.rst +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -454,8 +454,8 @@ event loop but the current event loop was set. .. nonce: humlhz .. section: Library -On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` -would include all six properties including ``processor``. +On :class:`os.uname_result`, restored expectation that ``_fields`` and +``_asdict`` would include all six properties including ``processor``. .. diff --git a/Misc/NEWS.d/3.13.0a3.rst b/Misc/NEWS.d/3.13.0a3.rst index 0f8dee261c65896..3d3b414c9d46af9 100644 --- a/Misc/NEWS.d/3.13.0a3.rst +++ b/Misc/NEWS.d/3.13.0a3.rst @@ -2166,7 +2166,7 @@ current user has no permission to the WMI. .. nonce: WOpiNt .. section: Windows -Deprecate :func:`sys._enablelegacywindowsfsencoding`. Use +Deprecate :func:`!sys._enablelegacywindowsfsencoding`. Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. Patch by Inada Naoki. .. diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst index d0eef2d8c11641a..1b7a28d84dcb7a6 100644 --- a/Misc/NEWS.d/3.13.0b1.rst +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -1429,7 +1429,7 @@ script) without specifying a value for ``UseTIER2``. .. nonce: LT27pF .. section: Build -The :file:`configure` option :option:`--with-system-libmpdec` now defaults +The :file:`configure` option :option:`!--with-system-libmpdec` now defaults to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python 3.15. @@ -1470,7 +1470,7 @@ Increase WASI stack size from 512 KiB to 8 MiB and the initial memory from .. section: Build :program:`configure` now uses :program:`pkg-config` to detect :mod:`decimal` -dependencies if the :option:`--with-system-libmpdec` option is given. +dependencies if the :option:`!--with-system-libmpdec` option is given. .. diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst index 5303bd89efff5ba..799369557572801 100644 --- a/Misc/NEWS.d/3.14.0a1.rst +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -2038,7 +2038,7 @@ Remove workarounds for non-IEEE 754 systems in :mod:`cmath`. .. nonce: WlygzR .. section: Library -Due to the lack of interest for :meth:`symtable.Class.get_methods`, the +Due to the lack of interest for :meth:`!symtable.Class.get_methods`, the method is marked as deprecated and will be removed in Python 3.16. Patch by Bénédikt Tran. @@ -2746,7 +2746,7 @@ situations. .. nonce: rRrprk .. section: Library -Fix :meth:`symtable.Class.get_methods` and document its behaviour. Patch by +Fix :meth:`!symtable.Class.get_methods` and document its behaviour. Patch by Bénédikt Tran. .. diff --git a/Misc/NEWS.d/3.15.0a1.rst b/Misc/NEWS.d/3.15.0a1.rst index f6c78879890fad7..c7e3a872fe72858 100644 --- a/Misc/NEWS.d/3.15.0a1.rst +++ b/Misc/NEWS.d/3.15.0a1.rst @@ -6395,7 +6395,7 @@ Add support for configuring compiler flags for the JIT with ``CFLAGS_JIT`` .. section: Build Removed implicit fallback to the bundled copy of the ``libmpdec`` library. -Now this should be explicitly enabled via :option:`--with-system-libmpdec` +Now this should be explicitly enabled via :option:`!--with-system-libmpdec` set to ``no`` or :option:`!--without-system-libmpdec`. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/3.15.0a8.rst b/Misc/NEWS.d/3.15.0a8.rst index ff7930aeb292d67..28e9ca85166620f 100644 --- a/Misc/NEWS.d/3.15.0a8.rst +++ b/Misc/NEWS.d/3.15.0a8.rst @@ -180,16 +180,6 @@ dealing with contradictions in ``make_bottom``. .. -.. date: 2026-03-24-13-06-52 -.. gh-issue: 146369 -.. nonce: 6wDI6S -.. section: Core and Builtins - -Ensure ``-X lazy_imports=none`` and ``PYTHON_LAZY_IMPORTS=none`` override -:attr:`~module.__lazy_modules__`. Patch by Hugo van Kemenade. - -.. - .. date: 2026-03-22-19-30-00 .. gh-issue: 146308 .. nonce: AxnRVA @@ -350,7 +340,7 @@ Fix :func:`repr` for lists and tuples containing ``NULL``\ s. .. nonce: aB3xKm .. section: Core and Builtins -Fixed ``sys.lazy_modules`` to include lazy modules without submodules. Patch +Fixed :py:attr:`sys.lazy_modules` to include lazy modules without submodules. Patch by Bartosz Sławecki. .. diff --git a/Misc/NEWS.d/3.15.0b1.rst b/Misc/NEWS.d/3.15.0b1.rst new file mode 100644 index 000000000000000..e0fcab0eef870fc --- /dev/null +++ b/Misc/NEWS.d/3.15.0b1.rst @@ -0,0 +1,2256 @@ +.. date: 2026-05-02-15-38-03 +.. gh-issue: 149254 +.. nonce: 0HOL0j +.. release date: 2026-05-07 +.. section: Security + +Update Android and iOS installer to use OpenSSL 3.5.6. + +.. + +.. date: 2026-04-26-17-49-58 +.. gh-issue: 149017 +.. nonce: EiVFPo +.. section: Security + +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.0. + +.. + +.. date: 2026-04-24-23-15-42 +.. gh-issue: 148252 +.. nonce: 8BLmzd +.. section: Security + +Fixed string table and sample record bounds checks in +:mod:`!_remote_debugging` when decoding certain ``.pyb`` inputs on 32-bit +builds. Patch by Maurycy Pawłowski-Wieroński. + +.. + +.. date: 2026-04-21-13-46-30 +.. gh-issue: 90309 +.. nonce: srvj9q +.. section: Security + +Base64-encode values when embedding cookies to JavaScript using the +:meth:`http.cookies.BaseCookie.js_output` method to avoid injection and +escaping. + +.. + +.. date: 2026-04-20-15-31-37 +.. gh-issue: 148808 +.. nonce: _Z8JL0 +.. section: Security + +Added buffer boundary check when using ``nbytes`` parameter with +:meth:`!asyncio.AbstractEventLoop.sock_recvfrom_into`. Only relevant for +Windows and the :class:`asyncio.ProactorEventLoop`. + +.. + +.. date: 2026-04-10-16-28-21 +.. gh-issue: 148395 +.. nonce: kfzm0G +.. section: Security + +Fix a dangling input pointer in :class:`lzma.LZMADecompressor`, +:class:`bz2.BZ2Decompressor`, and internal :class:`!zlib._ZlibDecompressor` +when memory allocation fails with :exc:`MemoryError`, which could let a +subsequent :meth:`!decompress` call read or write through a stale pointer to +the already-released caller buffer. + +.. + +.. date: 2026-04-08-14-25-47 +.. gh-issue: 148252 +.. nonce: IEp9Rt +.. section: Security + +Fixed stack depth calculation in :mod:`!_remote_debugging` when decoding +certain ``.pyb`` inputs on 32-bit builds. Issue originally identified and +diagnosed by Tristan Madani (@TristanInSec on GitHub). + +.. + +.. date: 2026-04-06-13-55-00 +.. gh-issue: 148178 +.. nonce: Rs7kLm +.. section: Security + +Hardened :mod:`!_remote_debugging` by validating remote debug offset tables +before using them to size memory reads or interpret remote layouts. + +.. + +.. date: 2026-03-31-09-15-51 +.. gh-issue: 148169 +.. nonce: EZJzz2 +.. section: Security + +A bypass in :mod:`webbrowser` allowed URLs prefixed with ``%action`` to pass +the dash-prefix safety check. + +.. + +.. date: 2026-03-29-12-51-33 +.. gh-issue: 146581 +.. nonce: 4vZfB0 +.. section: Security + +Fix vulnerability in :func:`shutil.unpack_archive` for ZIP files on Windows +which allowed to write files outside of the destination tree if the patch in +the archive contains a Windows drive prefix. Now such invalid paths will be +skipped. Files containing ".." in the name (like "foo..bar") are no longer +skipped. + +.. + +.. date: 2026-03-26-01-42-15 +.. gh-issue: 137586 +.. nonce: j3SkOm +.. section: Security + +Fix a PATH-injection vulnerability in :mod:`webbrowser` on macOS where +``osascript`` was invoked without an absolute path. The new :class:`!MacOS` +class uses ``/usr/bin/open`` directly, eliminating the dependency on +``osascript`` entirely. + +.. + +.. date: 2026-03-25-00-51-03 +.. gh-issue: 146333 +.. nonce: LqdL__bn +.. section: Security + +Fix quadratic backtracking in :class:`configparser.RawConfigParser` option +parsing regexes (``OPTCRE`` and ``OPTCRE_NV``). A crafted configuration line +with many whitespace characters could cause excessive CPU usage. + +.. + +.. date: 2026-03-20-09-29-42 +.. gh-issue: 146211 +.. nonce: PQVbs7 +.. section: Security + +Reject CR/LF characters in tunnel request headers for the +HTTPConnection.set_tunnel() method. + +.. + +.. date: 2026-05-06-15-57-28 +.. gh-issue: 148940 +.. nonce: dRIXiY +.. section: Core and Builtins + +Revert the process size based deferral of garbage collection (GH-133464). +The performance issue this change resolves is also fixed by GH-142562. This +approach has the problem that process size as seen by the OS (e.g. the +resident size or RSS) does not immediately decrease after cyclic garbage is +freed since mimalloc defers returning memory of the OS. This change applies +to the free-threaded GC only. + +.. + +.. date: 2026-05-03-10-24-50 +.. gh-issue: 149243 +.. nonce: Zh1q9_ +.. section: Core and Builtins + +Check for recursion limits in ``CALL_ALLOC_AND_ENTER_INIT`` opcode. + +.. + +.. date: 2026-05-02-18-02-41 +.. gh-issue: 126910 +.. nonce: nqDVrp +.. section: Core and Builtins + +Add support for unwinding JIT frames using GNU backtrace. Patch by Diego +Russo and Pablo Galindo + +.. + +.. date: 2026-04-30-01-35-09 +.. gh-issue: 149171 +.. nonce: meXWpl +.. section: Core and Builtins + +Allow assignment to the ``__module__`` attribute of +:class:`typing.TypeAliasType` instances. + +.. + +.. date: 2026-04-29-14-06-00 +.. gh-issue: 149122 +.. nonce: P8k2Lm +.. section: Core and Builtins + +Fix a crash in optimized calls to :func:`all`, :func:`any`, :func:`tuple`, +:func:`list`, and :func:`set` with an async generator expression argument +(for example, ``tuple(await x for x in y)``). These calls now correctly +raise ``TypeError`` instead of crashing. + +.. + +.. date: 2026-04-28-21-19-21 +.. gh-issue: 149049 +.. nonce: 98u2Ib +.. section: Core and Builtins + +Fix stack underflow for ``BINARY_OP`` in tier 2. + +.. + +.. date: 2026-04-28-05-59-17 +.. gh-issue: 83065 +.. nonce: f0UPNE +.. section: Core and Builtins + +Fix a deadlock that could occur when one thread is importing a submodule +(for example ``import pkg.sub.mod``) while another thread is importing one +of its parent packages (for example ``import pkg.sub``) and that parent's +``__init__.py`` itself imports the submodule. The import system now acquires +module locks in hierarchical (parent-before-child) order so the two threads +serialise instead of raising ``_DeadlockError``. + +.. + +.. date: 2026-04-22-14-55-18 +.. gh-issue: 113956 +.. nonce: 0VEXd6 +.. section: Core and Builtins + +Fix a data race in :func:`sys.intern` in the free-threaded build when +interning a string owned by another thread. An interned copy owned by the +current thread is used instead when it is not safe to immortalize the +original. + +.. + +.. date: 2026-04-21-19-29-29 +.. gh-issue: 148850 +.. nonce: MSH0J_ +.. section: Core and Builtins + +Fix the memory sanitizer false positive in :func:`os.getrandom`. + +.. + +.. date: 2026-04-21-14-36-44 +.. gh-issue: 148820 +.. nonce: XhOGhA +.. section: Core and Builtins + +Fix a race in :c:type:`!_PyRawMutex` on the free-threaded build where a +``Py_PARK_INTR`` return from ``_PySemaphore_Wait`` could let the waiter +destroy its semaphore before the unlocking thread's ``_PySemaphore_Wakeup`` +completed, causing a fatal ``ReleaseSemaphore`` error. + +.. + +.. date: 2026-04-21-06-43-32 +.. gh-issue: 148829 +.. nonce: GtIrYO +.. section: Core and Builtins + +Add :class:`sentinel`, implementing :pep:`661`. PEP by Tal Einat; patch by +Jelle Zijlstra. + +.. + +.. date: 2026-04-20-15-25-55 +.. gh-issue: 146270 +.. nonce: qZYfyc +.. section: Core and Builtins + +Fix a sequential consistency bug in ``structmember.c``. + +.. + +.. date: 2026-04-19-22-35-39 +.. gh-issue: 148766 +.. nonce: coLWln +.. section: Core and Builtins + +The interpreter help (such as ``python --help``) is now in color. Patch by +Hugo van Kemenade. + +.. + +.. date: 2026-04-18-16-41-04 +.. gh-issue: 148571 +.. nonce: Q6WB3A +.. section: Core and Builtins + +Fix a crash in the JIT optimizer when specialized opcode families inherited +incompatible recorded operand layouts. + +.. + +.. date: 2026-04-17-20-37-02 +.. gh-issue: 148653 +.. nonce: nbbHMh +.. section: Core and Builtins + +Forbid :mod:`marshalling <marshal>` recursive code objects, :class:`slice` +and :class:`frozendict` objects which cannot be correctly unmarshalled. + +.. + +.. date: 2026-04-17-11-30-00 +.. gh-issue: 142516 +.. nonce: GcGen315 +.. section: Core and Builtins + +Forward-port the generational cycle garbage collector to the default 3.15 +build, replacing the incremental collector while leaving the free-threaded +collector unchanged. + +.. + +.. date: 2026-04-15-12-00-00 +.. gh-issue: 146462 +.. nonce: 1YfK6v +.. section: Core and Builtins + +Added ``PyTypeObject.tp_basicsize``, ``PyTypeObject.tp_dictoffset``, and +``PyHeapTypeObject.ht_cached_keys`` offsets to :c:type:`!_Py_DebugOffsets` +to support version-independent read-only dict introspection tools. + +.. + +.. date: 2026-04-13-23-21-45 +.. gh-issue: 145239 +.. nonce: pL8qRt +.. section: Core and Builtins + +Unary plus is now accepted in :keyword:`match` literal patterns, mirroring +the existing support for unary minus. Patch by Bartosz Sławecki. + +.. + +.. date: 2026-04-13-16-52-33 +.. gh-issue: 148515 +.. nonce: 09xulC +.. section: Core and Builtins + +Fix a bug in the JIT optimizer reading operands for uops with multiple +caches. + +.. + +.. date: 2026-04-12-17-27-28 +.. gh-issue: 148390 +.. nonce: MAhw7F +.. section: Core and Builtins + +Fix an undefined behavior in :class:`memoryview` when using the native +boolean format (``?``) in :meth:`~memoryview.cast`. Previously, on some +common platforms, calling ``memoryview(b).cast("?").tolist()`` incorrectly +returned ``[False]`` instead of ``[True]`` for any even byte *b*. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-04-12-10-40-57 +.. gh-issue: 148418 +.. nonce: ggA1LZ +.. section: Core and Builtins + +Fix a possible reference leak in a corrupted ``TYPE_CODE`` marshal stream. + +.. + +.. date: 2026-04-11-17-28-52 +.. gh-issue: 148393 +.. nonce: lX6gwN +.. section: Core and Builtins + +Fix data races between :c:func:`PyDict_Watch` / :c:func:`PyDict_Unwatch` and +concurrent dict mutation in the :term:`free-threaded build`. + +.. + +.. date: 2026-04-11-15-12-53 +.. gh-issue: 148398 +.. nonce: g62jCA +.. section: Core and Builtins + +Fix a bug in the JIT optimizer where class attribute loads were not +invalidated after type mutation. + +.. + +.. date: 2026-04-10-23-13-19 +.. gh-issue: 146527 +.. nonce: P3Xv4Q +.. section: Core and Builtins + +Add a ``GCMonitor`` class with a ``get_gc_stats`` method to the +:mod:`!_remote_debugging` module to allow reading GC statistics from an +external Python process without requiring the full ``RemoteUnwinder`` +functionality. Patch by Sergey Miryanov and Pablo Galindo. + +.. + +.. date: 2026-04-10-14-20-54 +.. gh-issue: 148284 +.. nonce: HKs-S_ +.. section: Core and Builtins + +Fix high stack consumption in Python's interpreter loop on Clang 22 by +setting function limits for inlining when building with computed gotos. + +.. + +.. date: 2026-04-09-14-18-33 +.. gh-issue: 148037 +.. nonce: aP3CSX +.. section: Core and Builtins + +Remove critical section from :c:func:`!PyCode_Addr2Line` in free-threading. + +.. + +.. date: 2026-04-08-06-59-23 +.. gh-issue: 115802 +.. nonce: jqfZty +.. section: Core and Builtins + +Improve JIT code generation on Linux AArch64 by reducing the indirect call +to external symbols. Patch by Diego Russo. + +.. + +.. date: 2026-04-08-02-49-07 +.. gh-issue: 148189 +.. nonce: 0KpXID +.. section: Core and Builtins + +Repaired undercount of bytes in type-specific free lists reported by +sys._debugmallocstats(). For types that participate in cyclic garbage +collection, it was missing two pointers used by GC. + +.. + +.. date: 2026-04-07-20-37-23 +.. gh-issue: 148222 +.. nonce: uF4D4E +.. section: Core and Builtins + +Fix vectorcall support in :class:`types.GenericAlias` when the underlying +type does not support the vectorcall protocol. Fix possible leaks in +:class:`types.GenericAlias` and :class:`types.UnionType` in case of memory +error. + +.. + +.. date: 2026-04-07-20-21-44 +.. gh-issue: 148208 +.. nonce: JAxpDU +.. section: Core and Builtins + +Fix recursion depth leak in :c:func:`PyObject_Print` + +.. + +.. date: 2026-04-06-18-25-53 +.. gh-issue: 95004 +.. nonce: CQeT_H +.. section: Core and Builtins + +The specializing interpreter now specializes for :class:`enum.Enum` +improving performance and scaling in free-threading. Patch by Kumar Aditya. + +.. + +.. date: 2026-04-05-16-10-00 +.. gh-issue: 149202 +.. nonce: W8sQeR +.. section: Core and Builtins + +Enable frame pointers by default for GCC-compatible CPython builds, +including ``-mno-omit-leaf-frame-pointer``, ``-marm`` on 32-bit ARM, and/or +``-mbackchain`` on s390x platforms when the compiler supports them, so +profilers and debuggers can unwind native interpreter frames more reliably. +Users can pass :option:`--without-frame-pointers` to ``./configure`` to opt +out. + +.. + +.. date: 2026-04-02-17-52-33 +.. gh-issue: 148014 +.. nonce: 2Y6ND_ +.. section: Core and Builtins + +Accept a function name in :option:`-X presite <-X>` command line option and +:envvar:`PYTHON_PRESITE` environment variable. Patch by Victor Stinner. + +.. + +.. date: 2026-04-02-13-25-09 +.. gh-issue: 147998 +.. nonce: wnzkRT +.. section: Core and Builtins + +Fixed a memory leak in interpreter helper calls so cleanup works when an +operation falls across interpreter boundaries. Patch by Maurycy +Pawłowski-Wieroński. + +.. + +.. date: 2026-03-26-08-49-35 +.. gh-issue: 146455 +.. nonce: f54083a9 +.. section: Core and Builtins + +Fix O(N²) compile-time regression in constant folding after it was moved +from AST to CFG optimizer. + +.. + +.. date: 2026-03-25-12-00-00 +.. gh-issue: 146306 +.. nonce: B9f62e +.. section: Core and Builtins + +Specialize float true division in the tier 2 optimizer with inplace mutation +for uniquely-referenced operands. + +.. + +.. date: 2026-03-23-11-34-37 +.. gh-issue: 142186 +.. nonce: v8Yp3W +.. section: Core and Builtins + +Global :mod:`sys.monitoring` events can now be turned on and disabled on a +per code object basis. Returning ``DISABLE`` from a callback disables the +event for the entire code object (for the current tool). + +.. + +.. date: 2026-03-17-20-30-17 +.. gh-issue: 126910 +.. nonce: NaUwmD +.. section: Core and Builtins + +Add support for unwinding JIT frames using GDB. Patch by Diego Russo and +Pablo Galindo. + +.. + +.. date: 2026-03-16-17-29-22 +.. gh-issue: 146031 +.. nonce: 6nyB7C +.. section: Core and Builtins + +The unstable API _PyInterpreterState_SetEvalFrameFunc has a companion +function _PyInterpreterState_SetEvalFrameAllowSpecialization to specify if +specialization should be allowed. When this option is set to 1 the +specializer will turn Python -> Python calls into specialized opcodes which +the replacement interpreter loop can choose to respect and perform inlined +dispatch. + +.. + +.. date: 2026-02-26-21-22-34 +.. gh-issue: 145278 +.. nonce: DHkYqt +.. section: Core and Builtins + +The :mod:`encodings` is now partially frozen, including the ``aliases`` and +``utf_8`` submodules. + +The :mod:`linecache` is now frozen. + +.. + +.. date: 2026-02-18-16-53-26 +.. gh-issue: 134584 +.. nonce: a-O4sd +.. section: Core and Builtins + +Optimize and eliminate redundant ref-counting for ``MAKE_FUNCTION`` in the +JIT. + +.. + +.. date: 2026-01-15-13-37-21 +.. gh-issue: 143886 +.. nonce: 2gk5QC +.. section: Core and Builtins + +Reorder function annotations so positional-only arguments are returned +before other arguments. This fixes how :func:`functools.singledispatch` +registers functions with positional-only arguments. + +.. + +.. date: 2025-12-08-00-25-35 +.. gh-issue: 98894 +.. nonce: hKWyfqNx +.. section: Core and Builtins + +Restore ``function__entry`` and ``function__return`` DTrace/SystemTap probes +that were broken since Python 3.11. + +.. + +.. date: 2025-08-16-12-56-08 +.. gh-issue: 116021 +.. nonce: hMN9yw +.. section: Core and Builtins + +Support for creating instances of abstract AST nodes from the :mod:`ast` +module is deprecated and scheduled for removal in Python 3.20. Patch by +Brian Schubert. + +.. + +.. date: 2025-08-15-21-33-16 +.. gh-issue: 137814 +.. nonce: 6yRTeu +.. section: Core and Builtins + +Fix the ``__qualname__`` attribute of ``__annotate__`` functions on +functions. + +.. + +.. date: 2025-08-09-19-00-36 +.. gh-issue: 137600 +.. nonce: p_p6OU +.. section: Core and Builtins + +:mod:`ast`: The constructors of AST nodes now raise a :exc:`TypeError` when +a required argument is omitted or when a keyword argument that does not map +to a field on the AST node is passed. These cases had previously raised a +:exc:`DeprecationWarning` since Python 3.13. Patch by Brian Schubert. + +.. + +.. date: 2025-08-01-20-31-30 +.. gh-issue: 137293 +.. nonce: 4x3JbV +.. section: Core and Builtins + +Fix :exc:`SystemError` when searching ELF Files in :func:`sys.remote_exec`. + +.. + +.. date: 2025-06-10-17-30-55 +.. gh-issue: 135357 +.. nonce: sUXU1W +.. section: Core and Builtins + +Add support for :data:`!socket.SO_PASSRIGHTS` on Linux. + +.. + +.. date: 2025-05-26-10-03-18 +.. gh-issue: 134690 +.. nonce: mUMT16 +.. section: Core and Builtins + +Removed deprecated in :pep:`626` since Python 3.12 +:attr:`!codeobject.co_lnotab` from :class:`types.CodeType`. + +.. + +.. date: 2025-01-17-19-48-28 +.. gh-issue: 100239 +.. nonce: 7pbTEA +.. section: Core and Builtins + +Specialize ``BINARY_OP`` for concatenation of lists and tuples, and +propagate the result type through ``_BINARY_OP_EXTEND`` in the tier 2 +optimizer so that follow-up type guards can be eliminated. + +.. + +.. date: 2026-05-06-14-26-37 +.. gh-issue: 148823 +.. nonce: ySmOE4 +.. section: Library + +Defer the import of ``_colorize`` in ``argparse`` until needed for coloring +output. + +.. + +.. date: 2026-05-06-05-56-59 +.. gh-issue: 141560 +.. nonce: wlSQaW +.. section: Library + +Add an *annotation_format* parameter to :func:`inspect.getfullargspec`. + +.. + +.. date: 2026-05-05-13-12-58 +.. gh-issue: 139489 +.. nonce: a8qqIM +.. section: Library + +Add the :func:`xml.is_valid_text` function, which allows to check whether a +string can be used in the XML document. + +.. + +.. date: 2026-05-05-00-30-04 +.. gh-issue: 142389 +.. nonce: 4daLzc +.. section: Library + +Add backticks to stdlib argparse help to display in colour. Patch by Hugo +van Kemenade. + +.. + +.. date: 2026-05-04-19-28-48 +.. gh-issue: 149377 +.. nonce: WNlc8Y +.. section: Library + +Update bundled pip to 26.1.1 + +.. + +.. date: 2026-05-04-18-01-35 +.. gh-issue: 142389 +.. nonce: 4Faqpq +.. section: Library + +Add backtick markup support in :mod:`argparse` option help text to highlight +inline code when color output is enabled. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-05-04-16-26-33 +.. gh-issue: 148675 +.. nonce: xZwXa6 +.. section: Library + +Remove ``F`` and ``D`` formats from :mod:`array` and :class:`memoryview`. +Patch by Victor Stinner. + +.. + +.. date: 2026-05-04-04-06-36 +.. gh-issue: 149342 +.. nonce: d3CK-y +.. section: Library + +Fix :mod:`!_remote_debugging` binary writing so that sampling a thread whose +Python frame stack is empty (for example while it is in a C call or +mid-syscall) no longer raises ``RuntimeError("Invalid stack encoding +type")``, and so that ``BinaryWriter.total_samples`` after :meth:`!finalize` +or context-manager exit includes samples flushed from the RLE buffer. Patch +by Maurycy Pawłowski-Wieroński. + +.. + +.. date: 2026-05-04-00-51-32 +.. gh-issue: 149010 +.. nonce: BCp_8k +.. section: Library + +The ``inspect`` module CLI now reports as much information as it has +available for non-source modules when ``--details`` is specified, and +provides an error message rather than a traceback when ``--details`` is +omitted. It also reports improved information when the given target location +is not the target's defining location and when the given target is a data +value rather than a class or function definition. + +.. + +.. date: 2026-05-03-23-47-59 +.. gh-issue: 146609 +.. nonce: V9jqYf +.. section: Library + +Use :mod:`argparse` for colour help :mod:`timeit` CLI. Patch by Hugo van +Kemenade. + +.. + +.. date: 2026-05-03-23-29-34 +.. gh-issue: 142389 +.. nonce: SVYiSv +.. section: Library + +Add backticks for colour to regrtest and pdb's help description. Patch by +Hugo van Kemenade. + +.. + +.. date: 2026-05-03-17-32-24 +.. gh-issue: 144384 +.. nonce: q-8jSr +.. section: Library + +Lazily import :mod:`!_colorize`. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-05-03-12-00-00 +.. gh-issue: 149321 +.. nonce: fUaxrz +.. section: Library + +Fix import cycles exposed by running standard library modules with ``-X +lazy_imports=none``. + +.. + +.. date: 2026-05-03-01-49-57 +.. gh-issue: 145378 +.. nonce: rtyAWM +.. section: Library + +Generate consistent colors for :mod:`pdb` commands in :mod:`pdb` REPL. + +.. + +.. date: 2026-05-02-19-09-04 +.. gh-issue: 149296 +.. nonce: DuKF0j +.. section: Library + +Add a ``dump`` subcommand to :mod:`profiling.sampling` that prints a single +traceback-style snapshot of a running process's Python stack, including +per-thread status, source line highlighting, optional bytecode opcode names, +and async-aware task reconstruction. Patch by Pablo Galindo. + +.. + +.. date: 2026-05-02-18-23-50 +.. gh-issue: 143231 +.. nonce: oBbQb5 +.. section: Library + +A *module* attribute has been added to :class:`!warnings.WarningMessage`. + +.. + +.. date: 2026-05-02-15-58-08 +.. gh-issue: 148675 +.. nonce: b3ZNlj +.. section: Library + +:mod:`ctypes`: Change the :py:attr:`~ctypes._SimpleCData._type_` of +:class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and +:class:`~ctypes.c_longdouble_complex` from ``F``, ``D`` and ``G`` to ``Zf``, +``Zd`` and ``Zg`` for compatibility with numpy. Patch by Victor Stinner. + +.. + +.. date: 2026-05-02-12-30-35 +.. gh-issue: 148675 +.. nonce: cu2YFT +.. section: Library + +The :data:`array.typecodes` type changed from :class:`str` to :class:`tuple` +to support type codes longer than 1 character (``Zf`` and ``Zd``). Patch by +Victor Stinner. + +.. + +.. date: 2026-05-02-01-09-29 +.. gh-issue: 149221 +.. nonce: __KOks +.. section: Library + +Catch rare math domain error for :func:`random.binomialvariate`. + +.. + +.. date: 2026-05-01-16-45-31 +.. gh-issue: 149231 +.. nonce: x2nBEE +.. section: Library + +In :mod:`tomllib`, the number of parts in TOML keys is now limited. + +.. + +.. date: 2026-05-01-11-39-37 +.. gh-issue: 143231 +.. nonce: 0cOHET +.. section: Library + +:func:`unittest.TestCase.assertWarns` and +:func:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that +do not match the specified category or regex. Nested context managers are +now supported. + +.. + +.. date: 2026-05-01-10-20-27 +.. gh-issue: 149214 +.. nonce: btP546 +.. section: Library + +Fix :mod:`!_remote_debugging` misreading non-ASCII Unicode strings (Latin-1, +BMP and non-BMP) from a remote process. Filenames and function names that +contain non-ASCII characters are now reported correctly in stack traces, the +sampling profiler, and :mod:`asyncio` task introspection. + +.. + +.. date: 2026-04-30-18-56-23 +.. gh-issue: 149189 +.. nonce: mszW10 +.. section: Library + +:mod:`pprint` now uses modern defaults: ``indent=4`` and ``width=88``, and +the default ``compact=False`` output is now formatted similar to +pretty-printed :func:`json.dumps`, with opening parentheses and brackets +followed by a newline and the contents indented by one level. The *expand* +parameter, added in 3.15.0a8, has been removed; ``compact=False`` (the +default) now produces the former ``expand=True`` layout. Patch by Hugo van +Kemenade. + +.. + +.. date: 2026-04-30-14-21-26 +.. gh-issue: 149173 +.. nonce: KJqZm0 +.. section: Library + +Fix inverted :envvar:`PYTHON_BASIC_REPL` environment check in +``pdb._pyrepl_available``. + +.. + +.. date: 2026-04-29-16-11-27 +.. gh-issue: 149117 +.. nonce: yEeTYd +.. section: Library + +Fix :func:`runpy.run_module` and :func:`runpy.run_path` to set the +:attr:`~ImportError.name` attribute on the :exc:`ImportError` they raise. + +.. + +.. date: 2026-04-29-14-33-42 +.. gh-issue: 149148 +.. nonce: EaiYvk +.. section: Library + +:mod:`ensurepip`: Upgrade bundled pip to 26.1. This version fixes the +:cve:`2026-3219` vulnerability. Patch by Victor Stinner. + +.. + +.. date: 2026-04-29-13-08-46 +.. gh-issue: 149009 +.. nonce: rek3Tw +.. section: Library + +Validate that :mod:`profiling.sampling` binary profiles do not contain more +unique (thread, interpreter) pairs than declared in the header. Patch by +Maurycy Pawłowski-Wieroński. + +.. + +.. date: 2026-04-28-17-47-55 +.. gh-issue: 148292 +.. nonce: oIq3ml +.. section: Library + +:mod:`ssl`: Update :class:`ssl.SSLSocket` and :class:`ssl.SSLObject` for +OpenSSL 4. The classes now remember if they get a :exc:`ssl.SSLEOFError`. In +this case, following :meth:`~ssl.SSLSocket.read`, :meth:`!sendfile`, +:meth:`~ssl.SSLSocket.write`, and :meth:`~ssl.SSLSocket.do_handshake` calls +raise :exc:`ssl.SSLEOFError` without calling the underlying OpenSSL +function. Thanks to that, :class:`ssl.SSLSocket` behaves the same on all +OpenSSL versions on EOF. Patch by Victor Stinner. + +.. + +.. date: 2026-04-28-16-30-48 +.. gh-issue: 149085 +.. nonce: 5aNgBD +.. section: Library + +Add a *max_threads* keyword argument to :func:`faulthandler.dump_traceback`, +:func:`faulthandler.dump_traceback_later`, :func:`faulthandler.enable`, and +:func:`faulthandler.register`. + +.. + +.. date: 2026-04-28-16-25-40 +.. gh-issue: 148641 +.. nonce: aFgym0 +.. section: Library + +:func:`pkgutil.resolve_name` gets a new optional, keyword-only argument +called ``strict``. The default is ``False`` for backward compatibility. + +.. + +.. date: 2026-04-27-22-34-09 +.. gh-issue: 148093 +.. nonce: 9pWceM +.. section: Library + +Fix an out-of-bounds read of one byte in :func:`binascii.a2b_uu`. Raise +:exc:`binascii.Error`, instead of reading past the buffer end. + +.. + +.. date: 2026-04-27-20-15-54 +.. gh-issue: 149083 +.. nonce: BdrpU8 +.. section: Library + +:data:`dataclasses.MISSING` and :data:`dataclasses.KW_ONLY` are now +instances of :class:`sentinel`. + +.. + +.. date: 2026-04-27-17-12-11 +.. gh-issue: 148914 +.. nonce: i5C3kW +.. section: Library + +Fix memoization of in-band :class:`~pickle.PickleBuffer` in the Python +implementation of :mod:`pickle`. Previously, identical +:class:`!PickleBuffer`\ s did not preserve identity, and empty writable +:class:`!PickleBuffer` memoized an empty bytearray object in place of +``b''``, so the following references to ``b''`` were unpickled as an empty +bytearray object. + +.. + +.. date: 2026-04-26-23-01-50 +.. gh-issue: 149026 +.. nonce: Akk4Bc +.. section: Library + +Add colour to :mod:`pickletools` CLI output. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-04-25-18-09-16 +.. gh-issue: 148991 +.. nonce: AZ64Et +.. section: Library + +Add colour to :mod:`tokenize` CLI output. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-04-25-14-11-24 +.. gh-issue: 138907 +.. nonce: u21Wnh +.. section: Library + +Support :rfc:`9309` in :mod:`urllib.robotparser`. + +.. + +.. date: 2026-04-25-12-50-46 +.. gh-issue: 148981 +.. nonce: YMM4Y9 +.. section: Library + +Add *color* parameter to :func:`ast.dump`. + +.. + +.. date: 2026-04-25-12-04-27 +.. gh-issue: 148849 +.. nonce: Vk6yEW +.. section: Library + +Deprecate :meth:`http.cookies.Morsel.js_output` and +:meth:`http.cookies.BaseCookie.js_output`, which will be removed in Python +3.19. Use :meth:`http.cookies.Morsel.output` or +:meth:`http.cookies.BaseCookie.output` instead. + +.. + +.. date: 2026-04-25-11-56-05 +.. gh-issue: 146311 +.. nonce: iHWO0v +.. section: Library + +Add a *canonical* keyword-only parameter to the base16, base32, base64, +base85, ascii85, and Z85 decoders in :mod:`base64` and :mod:`binascii`. When +true, encodings with non-zero padding bits (base16/32/64) or non-canonical +encodings (base85/ascii85) are rejected. Single-character final groups in +:func:`binascii.a2b_ascii85` and :func:`binascii.a2b_base85` are now always +rejected as encoding violations, regardless of *canonical*; previously they +were silently ignored and produced no output bytes. + +.. + +.. date: 2026-04-23-21-47-49 +.. gh-issue: 148947 +.. nonce: W4V2lG +.. section: Library + +Fix crash in :deco:`dataclasses.dataclass` with ``slots=True`` that occurred +when a function found within the class had an empty ``__class__`` cell. + +.. + +.. date: 2026-04-23-07-38-04 +.. gh-issue: 148680 +.. nonce: ___ePl +.. section: Library + +``ForwardRef`` objects that contain internal names to represent known +objects now show the ``type_repr`` of the known object rather than the +internal ``__annotationlib_name_x__`` name when evaluated as strings. + +.. + +.. date: 2026-04-22-20-49-49 +.. gh-issue: 124397 +.. nonce: plMglV +.. section: Library + +The threading module added tooling to support concurrent iterator access: +:class:`threading.serialize_iterator`, +:func:`threading.synchronized_iterator`, and +:func:`threading.concurrent_tee`. + +.. + +.. date: 2026-04-20-18-29-21 +.. gh-issue: 148801 +.. nonce: ROeNqs +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +<object.__deepcopy__>` on deeply nested trees. + +.. + +.. date: 2026-04-18-21-39-15 +.. gh-issue: 148735 +.. nonce: siw6DG +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a use-after-free in +:meth:`Element.findtext <xml.etree.ElementTree.Element.findtext>` when the +element tree is mutated concurrently during the search. + +.. + +.. date: 2026-04-18-17-37-13 +.. gh-issue: 148740 +.. nonce: sYnFi0 +.. section: Library + +Fix usage for :mod:`uuid` command-line interface to support a custom +namespace be provided for uuid3 and uuid5. + +.. + +.. date: 2026-04-17-16-31-58 +.. gh-issue: 148688 +.. nonce: vVugFn +.. section: Library + +:mod:`bz2`, :mod:`compression.zstd`, :mod:`lzma`, :mod:`zlib`: Fix a double +free on memory allocation failure. Patch by Victor Stinner. + +.. + +.. date: 2026-04-17-13-56-44 +.. gh-issue: 148675 +.. nonce: f1kG70 +.. section: Library + +:mod:`array`, :mod:`struct`: Add support for ``Zd`` and ``Zf`` formats for +double complex and float complex. Patch by Victor Stinner. + +.. + +.. date: 2026-04-16-13-30-00 +.. gh-issue: 148651 +.. nonce: ZsTdLk +.. section: Library + +Fix reference leak in :class:`compression.zstd.ZstdDecompressor` when an +invalid option key is passed. + +.. + +.. date: 2026-04-15-21-46-52 +.. gh-issue: 148641 +.. nonce: -aoFyC +.. section: Library + +:pep:`829` (package startup configuration files) implements a new format +``<name>.start`` parallel to ``<name>.pth`` files, to replace ``import`` +lines in the latter. + +.. + +.. date: 2026-04-15-20-32-55 +.. gh-issue: 148639 +.. nonce: -dwsjB +.. section: Library + +Implement :pep:`800`, adding the :deco:`typing.disjoint_base` decorator. +Patch by Jelle Zijlstra. + +.. + +.. date: 2026-04-15-16-08-12 +.. gh-issue: 148615 +.. nonce: Uvx50R +.. section: Library + +Fix :mod:`pdb` to accept standard -- end of options separator. Reported by +haampie. Patched by Shrey Naithani. + +.. + +.. date: 2026-04-15-11-00-39 +.. gh-issue: 146553 +.. nonce: VGOsoP +.. section: Library + +Fix infinite loop in :func:`typing.get_type_hints` when ``__wrapped__`` +forms a cycle. Patch by Shamil Abdulaev. + +.. + +.. date: 2026-04-15-09-36-03 +.. gh-issue: 148599 +.. nonce: 90i1Ku +.. section: Library + +Update the :mod:`socket` module's WSA error messages to match official +documentation. + +.. + +.. date: 2026-04-14-09-04-35 +.. gh-issue: 148508 +.. nonce: -GiXml +.. section: Library + +An intermittent timing error when running SSL tests on iOS has been +resolved. + +.. + +.. date: 2026-04-13-21-38-50 +.. gh-issue: 144881 +.. nonce: 3kPqXw +.. section: Library + +:mod:`asyncio` debugging tools (``python -m asyncio ps`` and ``pstree``) now +retry automatically on transient errors that can occur when attaching to a +process under active thread delegation. The number of retries can be +controlled with the ``--retries`` flag. Patch by Bartosz Sławecki. + +.. + +.. date: 2026-04-13-15-59-44 +.. gh-issue: 148518 +.. nonce: RQdvsu +.. section: Library + +If an email containing an address header that ended in an open double quote +was parsed with a non-``compat32`` policy, accessing the ``username`` +attribute of the mailbox accessed through that header object would result in +an ``IndexError``. It now correctly returns an empty string as the result. + +.. + +.. date: 2026-04-13-06-22-27 +.. gh-issue: 148464 +.. nonce: Bj_NZy +.. section: Library + +Add missing ``__ctype_le/be__`` attributes for +:class:`~ctypes.c_float_complex` and :class:`~ctypes.c_double_complex`. +Patch by Sergey B Kirpichev. + +.. + +.. date: 2026-04-12-16-40-11 +.. gh-issue: 148370 +.. nonce: 0Li2EK +.. section: Library + +:mod:`configparser`: prevent quadratic behavior when a +:exc:`~configparser.ParsingError` is raised after a parser fails to parse +multiple lines. Patch by Bénédikt Tran. + +.. + +.. date: 2026-04-12-12-31-45 +.. gh-issue: 121190 +.. nonce: O6-E5_ +.. section: Library + +``importlib.resources.files()`` now emits a more meaningful error message +when module spec is None (as found in some ``__main__`` modules). + +.. + +.. date: 2026-04-11-17-28-06 +.. gh-issue: 127012 +.. nonce: h3rLYS +.. section: Library + +``importlib.abc.Traversable.read_text`` now allows/solicits an ``errors`` +parameter. + +.. + +.. date: 2026-04-11-12-32-38 +.. gh-issue: 137855 +.. nonce: tsVny_ +.. section: Library + +Improve import time of :mod:`dataclasses` module by lazy importing :mod:`re` +and :mod:`copy` modules. + +.. + +.. date: 2026-04-10-20-23-22 +.. gh-issue: 148352 +.. nonce: lrec3W +.. section: Library + +Add more color to :mod:`calendar`'s CLI output. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-04-09-12-42-42 +.. gh-issue: 148254 +.. nonce: Xt7vKs +.. section: Library + +Use singular "sec" instead of "secs" in :mod:`timeit` verbose output for +consistency with other time units. + +.. + +.. date: 2026-04-08-21-39-01 +.. gh-issue: 130472 +.. nonce: 4Bk6qH +.. section: Library + +Integrate fancycompleter with import completions. + +.. + +.. date: 2026-04-08-14-19-17 +.. gh-issue: 148241 +.. nonce: fO_QT4 +.. section: Library + +:mod:`json`: Fix serialization: no longer call ``str(obj)`` on :class:`str` +subclasses. Patch by Victor Stinner. + +.. + +.. date: 2026-04-08-11-44-12 +.. gh-issue: 148225 +.. nonce: H34yJp +.. section: Library + +The :mod:`profiling.sampling` ``replay`` command now rejects non-binary +profile files with a clear error explaining that replay only accepts files +created with ``--binary``. + +.. + +.. date: 2026-04-07-14-13-40 +.. gh-issue: 148192 +.. nonce: 34AUYQ +.. section: Library + +``email.generator.Generator._make_boundary`` could fail to detect a +duplicate boundary string if linesep was not \n. It now correctly detects +boundary strings when linesep is \r\n as well. + +.. + +.. date: 2026-04-07-12-37-53 +.. gh-issue: 148207 +.. nonce: YhGem4 +.. section: Library + +:class:`typing.TypeVarTuple` now accepts ``bound``, ``covariant``, +``contravariant``, and ``infer_variance`` parameters, matching the interface +of :class:`typing.TypeVar` and :class:`typing.ParamSpec`. + +.. + +.. date: 2026-04-04-20-22-02 +.. gh-issue: 148100 +.. nonce: lSmGQi +.. section: Library + +:term:`Soft deprecate <soft deprecated>` :func:`re.match` and +:meth:`re.Pattern.match` in favour of :func:`re.prefixmatch` and +:meth:`re.Pattern.prefixmatch`. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-04-02-05-06-34 +.. gh-issue: 147991 +.. nonce: 2ANtR5 +.. section: Library + +Improve :mod:`tomllib` import time (up to 10x faster). Patch by Victor +Stinner. + +.. + +.. date: 2026-04-01-07-10-49 +.. gh-issue: 147957 +.. nonce: QXf5Xx +.. section: Library + +Guarantees that :meth:`collections.UserDict.popitem` will pop in the same +order as the wrapped dictionary rather than an arbitrary order. + +.. + +.. date: 2026-03-31-17-33-10 +.. gh-issue: 146256 +.. nonce: Nm_Ke_ +.. section: Library + +The ``profiling.sampling`` module now supports JSONL output format via +``--jsonl``. Each run emits a newline-delimited JSON file that is +sequentially parseable by external tools, scripts, and programmatic +consumers. Patch by Maurycy Pawłowski-Wieroński. + +.. + +.. date: 2026-03-29-21-31-14 +.. gh-issue: 146609 +.. nonce: BnshCt +.. section: Library + +Add colour to :mod:`timeit` CLI output. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-03-28-11-31-32 +.. gh-issue: 146563 +.. nonce: cXtSym +.. section: Library + +:mod:`xml.parsers.expat`: add an exception note when a custom Expat handler +return value cannot be properly interpreted. Patch by Bénédikt Tran. + +.. + +.. date: 2026-03-26-01-42-20 +.. gh-issue: 137586 +.. nonce: KmHRwR +.. section: Library + +Add :class:`!MacOS` to :mod:`webbrowser` for macOS, which opens URLs via +``/usr/bin/open`` instead of piping AppleScript to ``osascript``. Deprecate +:class:`!MacOSXOSAScript` in favour of :class:`!MacOS`. + +.. + +.. date: 2026-03-25-07-17-41 +.. gh-issue: 146406 +.. nonce: ydsmqe +.. section: Library + +Cross-language method suggestions are now shown for :exc:`AttributeError` on +builtin types and their subclasses. For example, ``[].push()`` suggests +``append``, ``(1,2).append(3)`` suggests using a ``list``, ``None.keys()`` +suggests expecting a ``dict``, and ``1.0.__or__`` suggests using an ``int``. + +.. + +.. date: 2026-03-22-23-42-22 +.. gh-issue: 146313 +.. nonce: RtDeAd +.. section: Library + +Fix a deadlock in :mod:`multiprocessing`'s resource tracker where the parent +process could hang indefinitely in :func:`os.waitpid` during interpreter +shutdown if a child created via :func:`os.fork` still held the resource +tracker's pipe open. + +.. + +.. date: 2026-03-22-16-52-04 +.. gh-issue: 146292 +.. nonce: rJvvs0 +.. section: Library + +Add colour to :mod:`~http.server.BaseHTTPRequestHandler` logs, as used by +the :mod:`http.server` CLI. Patch by Hugo van Kemenade. + +.. + +.. date: 2026-03-13-14-23-33 +.. gh-issue: 145917 +.. nonce: TooGKx +.. section: Library + +Add MIME types for TTC and Haptics formats to :mod:`mimetypes`. (Contributed +by Charlie Lin in :gh:`145918`.) + +.. + +.. date: 2026-03-12-00-00-00 +.. gh-issue: 145846 +.. nonce: UbHxjv +.. section: Library + +Fix memory leak in ``_lsprof`` when ``clear()`` is called during active +profiling with nested calls. ``clearEntries()`` now walks the entire +``currentProfilerContext`` linked list instead of only freeing the top +context. + +.. + +.. date: 2026-03-11-15-09-52 +.. gh-issue: 145831 +.. nonce: _sW94w +.. section: Library + +Fix :func:`!email.quoprimime.decode` leaving a stray ``\r`` when +``eol='\r\n'`` by stripping the full *eol* string instead of one character. + +.. + +.. date: 2026-03-01-01-58-10 +.. gh-issue: 145378 +.. nonce: oy6rb9 +.. section: Library + +Use ``PyREPL`` as the default input console for :mod:`pdb` + +.. + +.. date: 2026-02-26-12-00-00 +.. gh-issue: 145244 +.. nonce: Kj31cp +.. section: Library + +Fixed a use-after-free in :mod:`json` encoder when a ``default`` callback +mutates the dictionary being serialized. + +.. + +.. date: 2026-02-25-22-20-00 +.. gh-issue: 117716 +.. nonce: w6kYp9 +.. section: Library + +Fix :mod:`wave` writing of odd-sized ``data`` chunks by appending the +required RIFF pad byte and correcting the RIFF chunk size field accordingly. + +.. + +.. date: 2026-02-25-10-00-00 +.. gh-issue: 145200 +.. nonce: m_4PAtcI +.. section: Library + +:mod:`hashlib`: fix a memory leak when allocating or initializing an OpenSSL +HMAC context fails. + +.. + +.. date: 2026-02-22-19-36-00 +.. gh-issue: 145056 +.. nonce: TH8nX4 +.. section: Library + +Add support for :class:`frozendict` in :meth:`dataclasses.asdict` and +:meth:`dataclasses.astuple`. + +.. + +.. date: 2026-02-22-00-00-00 +.. gh-issue: 145105 +.. nonce: csv-reader-reentrant +.. section: Library + +Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator +that calls :func:`next` on the same reader from within ``__next__``. + +.. + +.. date: 2026-02-19-04-40-57 +.. gh-issue: 130750 +.. nonce: 0hW52O +.. section: Library + +Restore quoting of choices in :mod:`argparse` error messages for improved +clarity and consistency with documentation. + +.. + +.. date: 2026-02-12-18-05-16 +.. gh-issue: 137855 +.. nonce: 2_PTbg +.. section: Library + +Reduce the import time of :mod:`dataclasses` module by ~20%. + +.. + +.. date: 2026-02-07-12-54-20 +.. gh-issue: 70647 +.. nonce: Bja_Lk +.. section: Library + +:meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the +format string contains ``%d`` without a year directive. Using ``%e`` without +a year now emits a :exc:`DeprecationWarning`. + +.. + +.. date: 2026-01-19-21-23-18 +.. gh-issue: 105936 +.. nonce: dGrzjM +.. section: Library + +Attempting to mutate non-field attributes of :mod:`dataclasses` with both +*frozen* and *slots* being ``True`` now raises +:class:`~dataclasses.FrozenInstanceError` instead of :class:`TypeError`. +Their non-dataclass subclasses can now freely mutate non-field attributes, +and the original non-slotted class can be garbage collected. + +.. + +.. date: 2025-12-17-04-10-35 +.. gh-issue: 142831 +.. nonce: ee3t4L +.. section: Library + +Fix a crash in the :mod:`json` module where a use-after-free could occur if +the object being encoded is modified during serialization. + +.. + +.. date: 2025-12-17-02-55-03 +.. gh-issue: 108411 +.. nonce: up7MAc +.. section: Library + +``typing.IO`` and ``typing.BinaryIO`` method arguments are now +positional-only. + +.. + +.. date: 2025-12-10-15-15-09 +.. gh-issue: 130273 +.. nonce: iCfiY5 +.. section: Library + +Fix traceback color output with Unicode characters. + +.. + +.. date: 2025-12-06-11-24-25 +.. gh-issue: 142307 +.. nonce: w8evI9 +.. section: Library + +:mod:`imaplib`: deprecate support for :attr:`IMAP4.file +<imaplib.IMAP4.file>`. This attribute was never meant to be part of the +public interface and altering its value may result in unclosed files or +other synchronization issues with the underlying socket. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-12-06-08-48-26 +.. gh-issue: 141449 +.. nonce: hQvNW_ +.. section: Library + +Improve tests and documentation for non-function callables as +:term:`annotate functions <annotate function>`. + +.. + +.. date: 2025-10-18-12-13-39 +.. gh-issue: 140287 +.. nonce: 49iU-4 +.. section: Library + +The :mod:`asyncio` REPL now handles exceptions when executing +:envvar:`PYTHONSTARTUP` scripts. Patch by Bartosz Sławecki. + +.. + +.. date: 2025-10-08-15-36-00 +.. gh-issue: 139489 +.. nonce: W46tvn +.. section: Library + +Add the :func:`xml.is_valid_name` function, which allows to check whether a +string can be used as an element or attribute name in XML. + +.. + +.. date: 2025-08-24-15-09-30 +.. gh-issue: 75707 +.. nonce: GOWZrC +.. section: Library + +Add optional ``mtime`` argument to :func:`tarfile.open`, for setting the +``mtime`` header field in ``.tar.gz`` archives. + +.. + +.. date: 2025-07-02-17-01-17 +.. gh-issue: 125862 +.. nonce: WgFYj3 +.. section: Library + +The :func:`contextlib.contextmanager` and +:func:`contextlib.asynccontextmanager` decorators now work correctly with +generators, coroutine functions, and async generators when the wrapped +callables are used as decorators. + +.. + +.. date: 2025-06-22-16-29-10 +.. gh-issue: 135528 +.. nonce: Rt_QhR +.. section: Library + +:mod:`http.cookiejar`: add "tv", "or", "nom", "sch", and "web" to the +default list of supported country code second-level domains. + +.. + +.. date: 2025-06-02-22-23-38 +.. gh-issue: 135056 +.. nonce: yz3dSs +.. section: Library + +Add a ``-H`` or ``--header`` CLI option to :program:`python -m http.server`. +Contributed by Anton I. Sipos. + +.. + +.. date: 2025-05-23-10-28-51 +.. gh-issue: 134551 +.. nonce: 0rnq0X +.. section: Library + +Add t-strings support to pprint functions + +.. + +.. date: 2025-05-16-01-43-58 +.. gh-issue: 133956 +.. nonce: 5kWDYd +.. section: Library + +Fix bug where :func:`@dataclass <dataclasses.dataclass>` wouldn't detect +``ClassVar`` fields if ``ClassVar`` was re-exported from a module other than +:mod:`typing`. + +.. + +.. date: 2025-04-17-15-26-35 +.. gh-issue: 132631 +.. nonce: IDFZfb +.. section: Library + +Fix "I/O operation on closed file" when parsing JSON Lines file with +:mod:`JSON CLI <json.tool>`. + +.. + +.. date: 2024-11-24-07-18-40 +.. gh-issue: 108951 +.. nonce: jyKygP +.. section: Library + +:mod:`asyncio`: Add :meth:`TaskGroup.cancel <asyncio.TaskGroup.cancel>` +which cancels unfinished tasks and exits the group without raising +:exc:`asyncio.CancelledError`. + +.. + +.. date: 2024-09-09-12-48-37 +.. gh-issue: 123853 +.. nonce: e-zFxb +.. section: Library + +Update the table of Windows language code identifiers (LCIDs) used by +:func:`locale.getdefaultlocale` on Windows to protocol version 16.0 +(2024-04-23). + +.. + +.. date: 2024-07-31-17-23-06 +.. gh-issue: 122476 +.. nonce: TtUa-c +.. section: Library + +The :mod:`email` module no longer incorrectly uses :rfc:`2047` encoding for +a mailbox with non-ASCII characters in its local-part. Under a policy with +:attr:`~email.policy.EmailPolicy.utf8` set ``False``, attempting to +serialize such a message will now raise an +:exc:`~email.errors.HeaderWriteError`. There is no valid 7-bit encoding for +an internationalized local-part. Use :data:`email.policy.SMTPUTF8` (or +another policy with ``utf8=True``) to correctly pass through the local-part +as Unicode characters. + +.. + +.. date: 2024-07-31-17-22-10 +.. gh-issue: 83938 +.. nonce: TtUa-c +.. section: Library + +The :mod:`email` module no longer incorrectly uses :rfc:`2047` encoding for +a mailbox with non-ASCII characters in its domain. Under a policy with +:attr:`~email.policy.EmailPolicy.utf8` set ``False``, attempting to +serialize such a message will now raise an +:exc:`~email.errors.HeaderWriteError`. Either apply an appropriate IDNA +encoding to convert the domain to ASCII before serialization, or use +:data:`email.policy.SMTPUTF8` (or another policy with ``utf8=True``) to +correctly pass through the internationalized domain name as Unicode +characters. + +.. + +.. date: 2024-07-30-19-19-33 +.. gh-issue: 81074 +.. nonce: YAeWNf +.. section: Library + +The :mod:`email` module no longer treats email addresses with non-ASCII +characters as defects when parsing a Unicode string or in the ``addr_spec`` +parameter to :class:`email.headerregistry.Address`. :rfc:`5322` permits such +addresses, and they were already supported when parsing bytes and in the +Address ``username`` parameter. + +The (undocumented) :exc:`!email.errors.NonASCIILocalPartDefect` is no longer +used and should be considered deprecated. + +.. + +.. date: 2024-02-10-21-25-22 +.. gh-issue: 70039 +.. nonce: 6wvcAP +.. section: Library + +Fixed bug where :meth:`smtplib.SMTP.starttls` could fail if +:meth:`smtplib.SMTP.connect` is called explicitly rather than implicitly. + +.. + +.. date: 2023-12-25-19-14-07 +.. gh-issue: 113471 +.. nonce: ZQMpbI +.. section: Library + +Allow :mod:`http.server` to set a default content-type when serving files +with an unknown or missing extension. + +.. + +.. date: 2023-09-08-13-10-32 +.. gh-issue: 83281 +.. nonce: 2Plpcj +.. section: Library + +:mod:`email`: improve handling trailing garbage in address lists to avoid +throwing AttributeError in certain edge cases + +.. + +.. date: 2022-09-17-20-20-01 +.. gh-issue: 96894 +.. nonce: t7my0A +.. section: Library + +Do not turn echo off for subsequent commands in batch activators +(``activate.bat`` and ``deactivate.bat``) of :mod:`venv`. + +.. + +.. date: 2026-04-17-02-28-55 +.. gh-issue: 148663 +.. nonce: MHIbRB +.. section: Documentation + +Document that :class:`calendar.IllegalMonthError` is a subclass of both +:exc:`ValueError` and :exc:`IndexError` since Python 3.12. + +.. + +.. date: 2026-04-02-07-20-00 +.. gh-issue: 146646 +.. nonce: GlobDoc1 +.. section: Documentation + +Document that :func:`glob.glob`, :func:`glob.iglob`, +:meth:`pathlib.Path.glob`, and :meth:`pathlib.Path.rglob` silently suppress +:exc:`OSError` exceptions raised from scanning the filesystem. + +.. + +.. date: 2026-05-05-18-49-44 +.. gh-issue: 149425 +.. nonce: QnQL8j +.. section: Tests + +Increase time delta in +``test.test_zipfile.test_core.OtherTests.test_write_without_source_date_epoch`` + +.. + +.. date: 2026-04-21-12-33-14 +.. gh-issue: 148600 +.. nonce: vnTb3t +.. section: Tests + +Add OpenSSL 4.0.0 support to test configurations. + +.. + +.. date: 2026-05-04-23-07-45 +.. gh-issue: 149353 +.. nonce: XfM8aQ +.. section: Build + +Avoid unnecessary JIT-related rebuilds during ``make install`` after +``--enable-optimizations`` builds. + +.. + +.. date: 2026-05-04-06-03-50 +.. gh-issue: 149351 +.. nonce: hN4sF0 +.. section: Build + +Avoid possible broken macOS framework install names when DESTDIR is +specified during builds. + +.. + +.. date: 2026-05-01-20-01-32 +.. gh-issue: 149252 +.. nonce: 4W_0-w +.. section: Build + +Update to WASI SDK 33. + +.. + +.. date: 2026-05-01-12-01-54 +.. gh-issue: 148690 +.. nonce: oTtYk- +.. section: Build + +Windows free-threaded builds now output to a different default path with +default filenames, for example, ``PCbuild/amd64t/python.exe`` rather than +``PCbuild/amd64/python3.15t.exe``. The ``PC/layout`` script has been updated +to ensure compatibility of generated layouts. + +.. + +.. date: 2026-04-30-08-43-47 +.. gh-issue: 146475 +.. nonce: 1cL4hX +.. section: Build + +Block Apple Clang from being used to build the JIT as it ships without +required LLVM tools. + +.. + +.. date: 2026-04-17-21-45-32 +.. gh-issue: 148644 +.. nonce: vwkknh +.. section: Build + +Errors during the PGO training job on Windows are no longer ignored, and a +non-zero return code will cause the build to fail. + +.. + +.. date: 2026-04-14-15-20-29 +.. gh-issue: 148535 +.. nonce: JjKiaa +.. section: Build + +No longer use the ``gcc -fprofile-update=atomic`` flag on i686. The flag has +been added to fix a random GCC internal error on PGO build (:gh:`145801`) +caused by corruption of profile data (.gcda files). The problem is that it +makes the PGO build way slower (up to 47x slower) on i686. Since the GCC +internal error was not seen on i686 so far, don't use +``-fprofile-update=atomic`` on i686 anymore. Patch by Victor Stinner. + +.. + +.. date: 2026-04-13-02-36-13 +.. gh-issue: 148483 +.. nonce: gLe1h8 +.. section: Build + +Use ``Py_GCC_ATTRIBUTE(unused)`` for stop_tracing label. + +.. + +.. date: 2026-04-12-22-54-16 +.. gh-issue: 148474 +.. nonce: ouIO8R +.. section: Build + +Fixed compilation of :file:`Python/pystrhex.c` with older clang versions. + +.. + +.. date: 2026-04-09-11-42-32 +.. gh-issue: 146445 +.. nonce: Z1vccC +.. section: Build + +The Android build tools have been moved to the Platforms folder. + +.. + +.. date: 2026-03-21-18-51-31 +.. gh-issue: 146264 +.. nonce: Q9Ej4m +.. section: Build + +Fix static module builds on non-WASI targets by linking HACL dependencies as +static libraries when ``MODULE_BUILDTYPE=static``, preventing duplicate +``_Py_LibHacl_*`` symbol errors at link time. + +.. + +.. date: 2025-09-03-14-55-59 +.. gh-issue: 138451 +.. nonce: -Qzh2S +.. section: Build + +Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT +build. + +.. + +.. date: 2025-05-02-17-06-10 +.. gh-issue: 133312 +.. nonce: YkO6BI +.. section: Build + +Add a new ``./configure`` option +:option:`--enable-static-libpython-for-interpreter` which, when used with +:option:`--enable-shared`, continues to build the shared library but does +not use it for the interpreter. Instead, libpython is statically linked into +the interpreter, as if :option:`--enable-shared` had not been used. This +allows you to do a single build and get a Python interpreter binary that +does not use a shared library but also get a shared library for use by other +programs. + +.. + +.. date: 2026-05-03-13-55-51 +.. gh-issue: 149254 +.. nonce: ENtMYD +.. section: Windows + +Updated bundled version of OpenSSL to 3.5.6. + +.. + +.. date: 2026-05-01-12-03-39 +.. gh-issue: 148690 +.. nonce: TMV8dU +.. section: Windows + +Non-freethreaded builds on Windows now support extensions linked to +``python3t.dll``, and will include a copy of that library in normal installs +that references the non-freethreaded runtime. + +.. + +.. date: 2026-03-27-22-06-10 +.. gh-issue: 146458 +.. nonce: fYj0UQ +.. section: Windows + +Fix incorrect REPL height and width tracking on console window resize on +Windows. + +.. + +.. date: 2026-05-06-18-23-36 +.. gh-issue: 142295 +.. nonce: O9RmZH +.. section: macOS + +For Python macOS framework builds, update Info.plist files to be more +compliant with current Apple guidelines. Original patch contributed by +Martinus Verburg. + +.. + +.. date: 2026-05-01-20-12-33 +.. gh-issue: 149254 +.. nonce: kXdWpS +.. section: macOS + +Update macOS installer to use OpenSSL 3.5.6. + +.. + +.. date: 2026-03-07-20-47-40 +.. gh-issue: 94523 +.. nonce: dq7m2k +.. section: IDLE + +Detect file if modified at local disk and prompt to ask refresh. Patch by +Shixian Li. + +.. + +.. date: 2025-10-05-19-33-39 +.. gh-issue: 139551 +.. nonce: TX9BRc +.. section: IDLE + +Support rendering :exc:`BaseExceptionGroup` in IDLE. + +.. + +.. date: 2021-10-03-21-55-34 +.. gh-issue: 89520 +.. nonce: etEExa +.. section: IDLE + +Make IDLE extension configuration look at user config files, allowing +user-installed extensions to have settings and key bindings defined in +~/.idlerc. + +.. + +.. date: 2026-05-01-14-49-09 +.. gh-issue: 149225 +.. nonce: IdAYPZ +.. section: C API + +:c:type:`PyCriticalSection` and related functions are added to the Stable +ABI. + +.. + +.. date: 2026-05-01-00-00-00 +.. gh-issue: 149216 +.. nonce: TpWatch +.. section: C API + +:c:type:`PyType_WatchCallback` callbacks registered via +:c:func:`PyType_AddWatcher` are now also invoked when a watched heap type is +deallocated. Previously, type watchers were only notified of modifications, +which could cause stale references when a type was freed and its address was +reused. + +.. + +.. date: 2026-04-28-17-43-12 +.. gh-issue: 149101 +.. nonce: HTuHTb +.. section: C API + +Implement :pep:`788`. + +.. + +.. date: 2026-04-27-10-56-22 +.. gh-issue: 149044 +.. nonce: TbOcUS +.. section: C API + +Implement :pep:`820`: Unified slot system for the C API. + +.. + +.. date: 2026-04-09-14-45-44 +.. gh-issue: 148267 +.. nonce: p84kG_ +.. section: C API + +Using :c:macro:`Py_LIMITED_API` on a non-Windows free-threaded build no +longer needs an extra :c:macro:`Py_GIL_DISABLED`. + +.. + +.. date: 2026-04-05-18-18-59 +.. gh-issue: 145559 +.. nonce: qKJH9S +.. section: C API + +Rename ``_Py_DumpTraceback`` and ``_Py_DumpTracebackThreads`` to +:c:func:`PyUnstable_DumpTraceback` and +:c:func:`PyUnstable_DumpTracebackThreads`. + +.. + +.. date: 2026-04-03-11-06-20 +.. gh-issue: 146636 +.. nonce: zR6Jsn +.. section: C API + +Implement :pep:`803` -- ``abi3t``: Stable ABI for Free-Threaded Builds. + +.. + +.. date: 2026-03-22-00-00-00 +.. gh-issue: 146302 +.. nonce: PyIsInit +.. section: C API + +:c:func:`Py_IsInitialized` no longer returns true until initialization has +fully completed, including import of the :mod:`site` module. The underlying +runtime flags now use atomic operations. + +.. + +.. date: 2026-03-19-15-28-14 +.. gh-issue: 146063 +.. nonce: Sc-1RU +.. section: C API + +Add :c:func:`PyObject_CallFinalizerFromDealloc` function to the limited C +API. Patch by Victor Stinner. + +.. + +.. date: 2026-03-13-16-37-54 +.. gh-issue: 145921 +.. nonce: ssA7HZ +.. section: C API + +Add functions that are guaranteed to be safe for use in +:c:member:`~PyTypeObject.tp_traverse` handlers: +:c:func:`PyObject_GetTypeData_DuringGC`, +:c:func:`PyObject_GetItemData_DuringGC`, +:c:func:`PyType_GetModuleState_DuringGC`, +:c:func:`PyModule_GetState_DuringGC`, :c:func:`PyModule_GetToken_DuringGC`, +:c:func:`PyType_GetBaseByToken_DuringGC`, +:c:func:`PyType_GetModule_DuringGC`, +:c:func:`PyType_GetModuleByToken_DuringGC`. diff --git a/Misc/NEWS.d/next/Build/2025-05-02-17-06-10.gh-issue-133312.YkO6BI.rst b/Misc/NEWS.d/next/Build/2025-05-02-17-06-10.gh-issue-133312.YkO6BI.rst deleted file mode 100644 index 3ad2db63907ecbf..000000000000000 --- a/Misc/NEWS.d/next/Build/2025-05-02-17-06-10.gh-issue-133312.YkO6BI.rst +++ /dev/null @@ -1,8 +0,0 @@ -Add a new ``./configure`` option -:option:`--enable-static-libpython-for-interpreter` which, when used -with :option:`--enable-shared`, continues to build the shared library -but does not use it for the interpreter. Instead, libpython is -statically linked into the interpreter, as if :option:`--enable-shared` -had not been used. This allows you to do a single build and get a Python -interpreter binary that does not use a shared library but also get a -shared library for use by other programs. diff --git a/Misc/NEWS.d/next/Build/2025-05-13-12-38-26.gh-issue-115119.7n0nx3.rst b/Misc/NEWS.d/next/Build/2025-05-13-12-38-26.gh-issue-115119.7n0nx3.rst new file mode 100644 index 000000000000000..0c6804ebf9c890d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-05-13-12-38-26.gh-issue-115119.7n0nx3.rst @@ -0,0 +1,2 @@ +Removed bundled copy of the libmpdec, use system library if it's available. +Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst b/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst deleted file mode 100644 index d83aee080255022..000000000000000 --- a/Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst +++ /dev/null @@ -1 +0,0 @@ -Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT build. diff --git a/Misc/NEWS.d/next/Build/2026-03-21-18-51-31.gh-issue-146264.Q9Ej4m.rst b/Misc/NEWS.d/next/Build/2026-03-21-18-51-31.gh-issue-146264.Q9Ej4m.rst deleted file mode 100644 index 1fdafe560432a66..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-03-21-18-51-31.gh-issue-146264.Q9Ej4m.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix static module builds on non-WASI targets by linking HACL dependencies as -static libraries when ``MODULE_BUILDTYPE=static``, preventing duplicate -``_Py_LibHacl_*`` symbol errors at link time. diff --git a/Misc/NEWS.d/next/Build/2026-04-09-11-42-32.gh-issue-146445.Z1vccC.rst b/Misc/NEWS.d/next/Build/2026-04-09-11-42-32.gh-issue-146445.Z1vccC.rst deleted file mode 100644 index e51454b9701e126..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-04-09-11-42-32.gh-issue-146445.Z1vccC.rst +++ /dev/null @@ -1 +0,0 @@ -The Android build tools have been moved to the Platforms folder. diff --git a/Misc/NEWS.d/next/Build/2026-04-12-22-54-16.gh-issue-148474.ouIO8R.rst b/Misc/NEWS.d/next/Build/2026-04-12-22-54-16.gh-issue-148474.ouIO8R.rst deleted file mode 100644 index d2d2bb62834572b..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-04-12-22-54-16.gh-issue-148474.ouIO8R.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed compilation of :file:`Python/pystrhex.c` with older clang versions. diff --git a/Misc/NEWS.d/next/Build/2026-04-13-02-36-13.gh-issue-148483.gLe1h8.rst b/Misc/NEWS.d/next/Build/2026-04-13-02-36-13.gh-issue-148483.gLe1h8.rst deleted file mode 100644 index 5b9b3069887580f..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-04-13-02-36-13.gh-issue-148483.gLe1h8.rst +++ /dev/null @@ -1 +0,0 @@ -Use ``Py_GCC_ATTRIBUTE(unused)`` for stop_tracing label. diff --git a/Misc/NEWS.d/next/Build/2026-04-14-15-20-29.gh-issue-148535.JjKiaa.rst b/Misc/NEWS.d/next/Build/2026-04-14-15-20-29.gh-issue-148535.JjKiaa.rst deleted file mode 100644 index 39f37acb14e098e..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-04-14-15-20-29.gh-issue-148535.JjKiaa.rst +++ /dev/null @@ -1,6 +0,0 @@ -No longer use the ``gcc -fprofile-update=atomic`` flag on i686. The flag has -been added to fix a random GCC internal error on PGO build (:gh:`145801`) -caused by corruption of profile data (.gcda files). The problem is that it -makes the PGO build way slower (up to 47x slower) on i686. Since the GCC -internal error was not seen on i686 so far, don't use -``-fprofile-update=atomic`` on i686 anymore. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst b/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst deleted file mode 100644 index a0cc9c9358cb264..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-04-17-21-45-32.gh-issue-148644.vwkknh.rst +++ /dev/null @@ -1 +0,0 @@ -Errors during the PGO training job on Windows are no longer ignored, and a non-zero return code will cause the build to fail. diff --git a/Misc/NEWS.d/next/Build/2026-04-21-17-27-52.gh-issue-148832.Kx9aQ1.rst b/Misc/NEWS.d/next/Build/2026-04-21-17-27-52.gh-issue-148832.Kx9aQ1.rst new file mode 100644 index 000000000000000..867cfcc5ab62ca2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-04-21-17-27-52.gh-issue-148832.Kx9aQ1.rst @@ -0,0 +1,4 @@ +Fix :option:`--enable-bolt` build by switching the default BOLT flag from +``-icf=1`` to ``-icf=0``. ``-icf=1`` folds address-taken functions and +breaks type-slot dispatch, crashing on :mod:`list` and :mod:`tuple` +concatenation. Patched by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Build/2026-04-30-08-43-47.gh-issue-146475.1cL4hX.rst b/Misc/NEWS.d/next/Build/2026-04-30-08-43-47.gh-issue-146475.1cL4hX.rst deleted file mode 100644 index 225c659393fac5c..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-04-30-08-43-47.gh-issue-146475.1cL4hX.rst +++ /dev/null @@ -1,2 +0,0 @@ -Block Apple Clang from being used to build the JIT as it ships without -required LLVM tools. diff --git a/Misc/NEWS.d/next/Build/2026-05-01-20-01-32.gh-issue-149252.4W_0-w.rst b/Misc/NEWS.d/next/Build/2026-05-01-20-01-32.gh-issue-149252.4W_0-w.rst deleted file mode 100644 index 646a8e337320169..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-05-01-20-01-32.gh-issue-149252.4W_0-w.rst +++ /dev/null @@ -1 +0,0 @@ -Update to WASI SDK 33. diff --git a/Misc/NEWS.d/next/Build/2026-05-04-06-03-50.gh-issue-149351.hN4sF0.rst b/Misc/NEWS.d/next/Build/2026-05-04-06-03-50.gh-issue-149351.hN4sF0.rst deleted file mode 100644 index 792c8d394eccb25..000000000000000 --- a/Misc/NEWS.d/next/Build/2026-05-04-06-03-50.gh-issue-149351.hN4sF0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid possible broken macOS framework install names when DESTDIR is -specified during builds. diff --git a/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst b/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst new file mode 100644 index 000000000000000..8248c24cbd511ac --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst @@ -0,0 +1,3 @@ +On Linux when Python is linked to the musl C library, use a thread stack +size of at least 1 MiB instead of musl default which is 128 kiB. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2026-05-19-12-45-23.gh-issue-131372.oJykeB.rst b/Misc/NEWS.d/next/Build/2026-05-19-12-45-23.gh-issue-131372.oJykeB.rst new file mode 100644 index 000000000000000..912f4131af27dae --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-19-12-45-23.gh-issue-131372.oJykeB.rst @@ -0,0 +1,3 @@ +Add a :option:`--with-build-details-suffix` configure flag to allow Linux +distributions that co-install multiple versions of Python in the same tree +to avoid ``build-details.json`` clashes. diff --git a/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst b/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst new file mode 100644 index 000000000000000..861261dd97269f9 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-21-15-14-59.gh-issue-148294.VtFaW4.rst @@ -0,0 +1,2 @@ +Corrected the use of ``AC_PATH_TOOL`` in ``configure.ac`` to allow a C++ +compiler to be found on :envvar:`!PATH`. diff --git a/Misc/NEWS.d/next/Build/2026-05-27-14-28-36.gh-issue-115119.wotFWh.rst b/Misc/NEWS.d/next/Build/2026-05-27-14-28-36.gh-issue-115119.wotFWh.rst new file mode 100644 index 000000000000000..04fdf3617f0cdef --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-27-14-28-36.gh-issue-115119.wotFWh.rst @@ -0,0 +1 @@ +Fix the detection of libmpdec header when no .pc files are available. diff --git a/Misc/NEWS.d/next/Build/2026-06-09-11-54-13.gh-issue-151163.vFAtjv.rst b/Misc/NEWS.d/next/Build/2026-06-09-11-54-13.gh-issue-151163.vFAtjv.rst new file mode 100644 index 000000000000000..e4f3a044c81c6f5 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-06-09-11-54-13.gh-issue-151163.vFAtjv.rst @@ -0,0 +1 @@ +Updated Android build to include SQLite version 3.53.2. diff --git a/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst b/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst new file mode 100644 index 000000000000000..98a8c2687357265 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-02-25-13-37-10.gh-issue-145235.-1ySNR.rst @@ -0,0 +1,3 @@ +Made :c:func:`PyDict_AddWatcher`, :c:func:`PyDict_ClearWatcher`, +:c:func:`PyDict_Watch`, and :c:func:`PyDict_Unwatch` thread-safe on the +:term:`free threaded <free threading>` build. diff --git a/Misc/NEWS.d/next/C_API/2026-03-13-16-37-54.gh-issue-145921.ssA7HZ.rst b/Misc/NEWS.d/next/C_API/2026-03-13-16-37-54.gh-issue-145921.ssA7HZ.rst deleted file mode 100644 index 4a5128320277035..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-03-13-16-37-54.gh-issue-145921.ssA7HZ.rst +++ /dev/null @@ -1,9 +0,0 @@ -Add functions that are guaranteed to be safe for use in -:c:member:`~PyTypeObject.tp_traverse` handlers: -:c:func:`PyObject_GetTypeData_DuringGC`, -:c:func:`PyObject_GetItemData_DuringGC`, -:c:func:`PyType_GetModuleState_DuringGC`, -:c:func:`PyModule_GetState_DuringGC`, :c:func:`PyModule_GetToken_DuringGC`, -:c:func:`PyType_GetBaseByToken_DuringGC`, -:c:func:`PyType_GetModule_DuringGC`, -:c:func:`PyType_GetModuleByToken_DuringGC`. diff --git a/Misc/NEWS.d/next/C_API/2026-03-19-15-28-14.gh-issue-146063.Sc-1RU.rst b/Misc/NEWS.d/next/C_API/2026-03-19-15-28-14.gh-issue-146063.Sc-1RU.rst deleted file mode 100644 index e20e11a754f6942..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-03-19-15-28-14.gh-issue-146063.Sc-1RU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :c:func:`PyObject_CallFinalizerFromDealloc` function to the limited C API. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-03-22-00-00-00.gh-issue-146302.PyIsInit.rst b/Misc/NEWS.d/next/C_API/2026-03-22-00-00-00.gh-issue-146302.PyIsInit.rst deleted file mode 100644 index e194e2bb2c378b6..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-03-22-00-00-00.gh-issue-146302.PyIsInit.rst +++ /dev/null @@ -1,3 +0,0 @@ -:c:func:`Py_IsInitialized` no longer returns true until initialization has -fully completed, including import of the :mod:`site` module. The underlying -runtime flags now use atomic operations. diff --git a/Misc/NEWS.d/next/C_API/2026-04-03-11-06-20.gh-issue-146636.zR6Jsn.rst b/Misc/NEWS.d/next/C_API/2026-04-03-11-06-20.gh-issue-146636.zR6Jsn.rst deleted file mode 100644 index 7f84a6f954dc76f..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-04-03-11-06-20.gh-issue-146636.zR6Jsn.rst +++ /dev/null @@ -1 +0,0 @@ -Implement :pep:`803` -- ``abi3t``: Stable ABI for Free-Threaded Builds. diff --git a/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst b/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst deleted file mode 100644 index 1ec1afd2cbfeb99..000000000000000 --- a/Misc/NEWS.d/next/C_API/2026-04-09-14-45-44.gh-issue-148267.p84kG_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Using :c:macro:`Py_LIMITED_API` on a non-Windows free-threaded build no -longer needs an extra :c:macro:`Py_GIL_DISABLED`. diff --git a/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst b/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst new file mode 100644 index 000000000000000..97721430edbd69d --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-05-12-16-47-21.gh-issue-149725.HZLBTZ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PySentinel_CheckExact` for exact :class:`sentinel` type tests +to accompany the existing :c:func:`PySentinel_Check`. diff --git a/Misc/NEWS.d/next/C_API/2026-05-31-15-39-58.gh-issue-150671.bTkDVC.rst b/Misc/NEWS.d/next/C_API/2026-05-31-15-39-58.gh-issue-150671.bTkDVC.rst new file mode 100644 index 000000000000000..a67a0f85db1a506 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-05-31-15-39-58.gh-issue-150671.bTkDVC.rst @@ -0,0 +1,4 @@ +Deprecate these C-API functions: :c:func:`PyGen_New`, +:c:func:`PyGen_NewWithQualName`, :c:func:`PyCoro_New`, and +:c:func:`PyAsyncGen_New`. +Schedule them for removal in 3.18 diff --git a/Misc/NEWS.d/next/C_API/2026-06-04-14-26-17.gh-issue-150907.CA91_B.rst b/Misc/NEWS.d/next/C_API/2026-06-04-14-26-17.gh-issue-150907.CA91_B.rst new file mode 100644 index 000000000000000..f58b248f3a0b986 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-06-04-14-26-17.gh-issue-150907.CA91_B.rst @@ -0,0 +1,2 @@ +Fix ``dynamic_annotations.h`` header file when built with C++ and Valgrind: +add ``extern "C++" scope`` for the C++ template. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-06-09-14-50-35.gh-issue-80384.UEGyvB.rst b/Misc/NEWS.d/next/C_API/2026-06-09-14-50-35.gh-issue-80384.UEGyvB.rst new file mode 100644 index 000000000000000..c4f5e8f3f9bbc2d --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-06-09-14-50-35.gh-issue-80384.UEGyvB.rst @@ -0,0 +1,3 @@ +:c:func:`PyWeakref_NewRef` and :c:func:`PyWeakref_NewProxy` now raise +:exc:`TypeError` if the *callback* argument is not callable, ``None``, or +``NULL``. diff --git a/Misc/NEWS.d/next/C_API/2026-06-10-15-22-44.gh-issue-149044.O7KEcs.rst b/Misc/NEWS.d/next/C_API/2026-06-10-15-22-44.gh-issue-149044.O7KEcs.rst new file mode 100644 index 000000000000000..fe0730b1bf87c4d --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-06-10-15-22-44.gh-issue-149044.O7KEcs.rst @@ -0,0 +1,3 @@ +Improved error message when specifying non-type base classes in +:c:macro:`Py_tp_bases`, :c:macro:`Py_tp_base`, and *bases* argument to +:c:func:`PyType_FromMetaclass` and other ``PyType_From*`` functions. diff --git a/Misc/NEWS.d/next/C_API/2026-06-10-16-43-37.gh-issue-123619.dV82r6.rst b/Misc/NEWS.d/next/C_API/2026-06-10-16-43-37.gh-issue-123619.dV82r6.rst new file mode 100644 index 000000000000000..4d4c94563330c06 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-06-10-16-43-37.gh-issue-123619.dV82r6.rst @@ -0,0 +1,3 @@ +:c:func:`PyUnstable_Object_EnableDeferredRefcount` now returns ``0`` if the +object is not tracked by the garbage collector: if :func:`gc.is_tracked` is +false. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-06-18-18-24-11.gh-issue-141510.-EOHJ1.rst b/Misc/NEWS.d/next/C_API/2026-06-18-18-24-11.gh-issue-141510.-EOHJ1.rst new file mode 100644 index 000000000000000..c77b462e97bdd1d --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-06-18-18-24-11.gh-issue-141510.-EOHJ1.rst @@ -0,0 +1 @@ +Add :class:`frozendict` to the fast paths of :c:func:`PyMapping_GetOptionalItem`, :c:func:`PyMapping_Keys`, :c:func:`PyMapping_Values`, and :c:func:`PyMapping_Items`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2019-12-27-19-20-33.bpo-38131.J5fv54.rst b/Misc/NEWS.d/next/Core_and_Builtins/2019-12-27-19-20-33.bpo-38131.J5fv54.rst new file mode 100644 index 000000000000000..8b606c4335e033a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2019-12-27-19-20-33.bpo-38131.J5fv54.rst @@ -0,0 +1,2 @@ +Produce more meaningful messages when compiling AST objects with wrong field +values. Patch by Batuhan Taskaya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-17-19-48-28.gh-issue-100239.7pbTEA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-01-17-19-48-28.gh-issue-100239.7pbTEA.rst deleted file mode 100644 index 594ef72ac57faef..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-01-17-19-48-28.gh-issue-100239.7pbTEA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Specialize ``BINARY_OP`` for concatenation of lists and tuples, and -propagate the result type through ``_BINARY_OP_EXTEND`` in the tier 2 -optimizer so that follow-up type guards can be eliminated. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-10-03-18.gh-issue-134690.mUMT16.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-10-03-18.gh-issue-134690.mUMT16.rst deleted file mode 100644 index d26fa590b3535f9..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-26-10-03-18.gh-issue-134690.mUMT16.rst +++ /dev/null @@ -1,2 +0,0 @@ -Removed deprecated in :pep:`626` since Python 3.12 -:attr:`!codeobject.co_lnotab` from :class:`types.CodeType`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-10-17-30-55.gh-issue-135357.sUXU1W.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-10-17-30-55.gh-issue-135357.sUXU1W.rst deleted file mode 100644 index 378bb59de7930a0..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-10-17-30-55.gh-issue-135357.sUXU1W.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for :data:`!socket.SO_PASSRIGHTS` on Linux. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-19-00-36.gh-issue-137600.p_p6OU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-19-00-36.gh-issue-137600.p_p6OU.rst deleted file mode 100644 index d1d1e36215daa36..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-09-19-00-36.gh-issue-137600.p_p6OU.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`ast`: The constructors of AST nodes now raise a :exc:`TypeError` when -a required argument is omitted or when a keyword argument that does not map to -a field on the AST node is passed. These cases had previously raised a -:exc:`DeprecationWarning` since Python 3.13. Patch by Brian Schubert. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-15-21-33-16.gh-issue-137814.6yRTeu.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-15-21-33-16.gh-issue-137814.6yRTeu.rst deleted file mode 100644 index 83561312deeb02c..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-15-21-33-16.gh-issue-137814.6yRTeu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix the ``__qualname__`` attribute of ``__annotate__`` functions on -functions. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst deleted file mode 100644 index 967d8faaef34226..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support for creating instances of abstract AST nodes from the :mod:`ast` module -is deprecated and scheduled for removal in Python 3.20. Patch by Brian Schubert. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-15-13-37-21.gh-issue-143886.2gk5QC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-15-13-37-21.gh-issue-143886.2gk5QC.rst deleted file mode 100644 index fe4835ec28cfd58..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-15-13-37-21.gh-issue-143886.2gk5QC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Reorder function annotations so positional-only arguments are returned -before other arguments. This fixes how :func:`functools.singledispatch` -registers functions with positional-only arguments. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-18-16-53-26.gh-issue-134584.a-O4sd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-18-16-53-26.gh-issue-134584.a-O4sd.rst deleted file mode 100644 index 1258efcc01581a5..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-18-16-53-26.gh-issue-134584.a-O4sd.rst +++ /dev/null @@ -1 +0,0 @@ -Optimize and eliminate redundant ref-counting for ``MAKE_FUNCTION`` in the JIT. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst deleted file mode 100644 index 6b6a9eb98136632..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-26-21-22-34.gh-issue-145278.DHkYqt.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :mod:`encodings` is now partially frozen, including -the ``aliases`` and ``utf_8`` submodules. - -The :mod:`linecache` is now frozen. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-16-17-29-22.gh-issue-146031.6nyB7C.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-16-17-29-22.gh-issue-146031.6nyB7C.rst deleted file mode 100644 index cabcf975e5aa89e..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-16-17-29-22.gh-issue-146031.6nyB7C.rst +++ /dev/null @@ -1 +0,0 @@ -The unstable API _PyInterpreterState_SetEvalFrameFunc has a companion function _PyInterpreterState_SetEvalFrameAllowSpecialization to specify if specialization should be allowed. When this option is set to 1 the specializer will turn Python -> Python calls into specialized opcodes which the replacement interpreter loop can choose to respect and perform inlined dispatch. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-20-30-17.gh-issue-126910.NaUwmD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-20-30-17.gh-issue-126910.NaUwmD.rst deleted file mode 100644 index 4d2634d0dd1e813..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-17-20-30-17.gh-issue-126910.NaUwmD.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for unwinding JIT frames using GDB. Patch by Diego Russo and Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst deleted file mode 100644 index 4a04658551c4441..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-23-11-34-37.gh-issue-142186.v8Yp3W.rst +++ /dev/null @@ -1,3 +0,0 @@ -Global :mod:`sys.monitoring` events can now be turned on and disabled on a -per code object basis. Returning ``DISABLE`` from a callback disables the -event for the entire code object (for the current tool). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-25-12-00-00.gh-issue-146306.B9f62e.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-25-12-00-00.gh-issue-146306.B9f62e.rst deleted file mode 100644 index c819a830f8bc86f..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-25-12-00-00.gh-issue-146306.B9f62e.rst +++ /dev/null @@ -1,2 +0,0 @@ -Specialize float true division in the tier 2 optimizer with inplace -mutation for uniquely-referenced operands. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-08-49-35.gh-issue-146455.f54083a9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-08-49-35.gh-issue-146455.f54083a9.rst deleted file mode 100644 index 4d7537f2529da6b..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-26-08-49-35.gh-issue-146455.f54083a9.rst +++ /dev/null @@ -1 +0,0 @@ -Fix O(N²) compile-time regression in constant folding after it was moved from AST to CFG optimizer. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-16-15-15.gh-issue-75723.BZ4Rsn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-16-15-15.gh-issue-75723.BZ4Rsn.rst new file mode 100644 index 000000000000000..596ca89958c9ed4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-03-31-16-15-15.gh-issue-75723.BZ4Rsn.rst @@ -0,0 +1 @@ +Avoid re-executing ``.pth`` files when :func:`site.addsitedir` is called for a known directory. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-02-13-25-09.gh-issue-147998.wnzkRT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-02-13-25-09.gh-issue-147998.wnzkRT.rst deleted file mode 100644 index e83d9929eae5e02..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-02-13-25-09.gh-issue-147998.wnzkRT.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a memory leak in interpreter helper calls so cleanup works when an -operation falls across interpreter boundaries. Patch by Maurycy -Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-02-17-52-33.gh-issue-148014.2Y6ND_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-02-17-52-33.gh-issue-148014.2Y6ND_.rst deleted file mode 100644 index 964e5bdabbe1f02..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-02-17-52-33.gh-issue-148014.2Y6ND_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Accept a function name in :option:`-X presite <-X>` command line option and -:envvar:`PYTHON_PRESITE` environment variable. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst deleted file mode 100644 index f82ca91f5ba0009..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst +++ /dev/null @@ -1,4 +0,0 @@ -Enable frame pointers by default for GCC-compatible CPython builds, including -``-mno-omit-leaf-frame-pointer`` when the compiler supports it, so profilers -and debuggers can unwind native interpreter frames more reliably. Users can pass -``--without-frame-pointers`` to opt out. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-06-18-25-53.gh-issue-95004.CQeT_H.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-06-18-25-53.gh-issue-95004.CQeT_H.rst deleted file mode 100644 index a492982bc62da7d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-06-18-25-53.gh-issue-95004.CQeT_H.rst +++ /dev/null @@ -1 +0,0 @@ -The specializing interpreter now specializes for :class:`enum.Enum` improving performance and scaling in free-threading. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-21-44.gh-issue-148208.JAxpDU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-21-44.gh-issue-148208.JAxpDU.rst deleted file mode 100644 index b8ae19f5877a7dc..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-21-44.gh-issue-148208.JAxpDU.rst +++ /dev/null @@ -1 +0,0 @@ -Fix recursion depth leak in :c:func:`PyObject_Print` diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst deleted file mode 100644 index 2c273fc4daba3de..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-07-20-37-23.gh-issue-148222.uF4D4E.rst +++ /dev/null @@ -1 +0,0 @@ -Fix vectorcall support in :class:`types.GenericAlias` when the underlying type does not support the vectorcall protocol. Fix possible leaks in :class:`types.GenericAlias` and :class:`types.UnionType` in case of memory error. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-08-02-49-07.gh-issue-148189.0KpXID.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-08-02-49-07.gh-issue-148189.0KpXID.rst deleted file mode 100644 index d90e30b3d3fba82..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-08-02-49-07.gh-issue-148189.0KpXID.rst +++ /dev/null @@ -1 +0,0 @@ -Repaired undercount of bytes in type-specific free lists reported by sys._debugmallocstats(). For types that participate in cyclic garbage collection, it was missing two pointers used by GC. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-08-06-59-23.gh-issue-115802.jqfZty.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-08-06-59-23.gh-issue-115802.jqfZty.rst deleted file mode 100644 index 13ed51be0e6c5c1..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-08-06-59-23.gh-issue-115802.jqfZty.rst +++ /dev/null @@ -1 +0,0 @@ -Improve JIT code generation on Linux AArch64 by reducing the indirect call to external symbols. Patch by Diego Russo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-09-14-18-33.gh-issue-148037.aP3CSX.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-09-14-18-33.gh-issue-148037.aP3CSX.rst deleted file mode 100644 index b0cef5951298172..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-09-14-18-33.gh-issue-148037.aP3CSX.rst +++ /dev/null @@ -1 +0,0 @@ -Remove critical section from :c:func:`!PyCode_Addr2Line` in free-threading. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-10-14-20-54.gh-issue-148284.HKs-S_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-10-14-20-54.gh-issue-148284.HKs-S_.rst deleted file mode 100644 index a74f6c1a61affd8..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-10-14-20-54.gh-issue-148284.HKs-S_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix high stack consumption in Python's interpreter loop on Clang 22 by setting function limits for inlining when building with computed gotos. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-10-23-13-19.gh-issue-146527.P3Xv4Q.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-10-23-13-19.gh-issue-146527.P3Xv4Q.rst deleted file mode 100644 index e83b66f71da0cae..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-10-23-13-19.gh-issue-146527.P3Xv4Q.rst +++ /dev/null @@ -1,5 +0,0 @@ -Add a ``GCMonitor`` class with a ``get_gc_stats`` method to the -:mod:`!_remote_debugging` module to allow reading GC statistics from an -external Python process without requiring the full ``RemoteUnwinder`` -functionality. -Patch by Sergey Miryanov and Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-11-15-12-53.gh-issue-148398.g62jCA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-11-15-12-53.gh-issue-148398.g62jCA.rst deleted file mode 100644 index 0a7ba0b27d97087..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-11-15-12-53.gh-issue-148398.g62jCA.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in the JIT optimizer where class attribute loads were not invalidated after type mutation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-11-17-28-52.gh-issue-148393.lX6gwN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-11-17-28-52.gh-issue-148393.lX6gwN.rst deleted file mode 100644 index 33c4b75bfb944ce..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-11-17-28-52.gh-issue-148393.lX6gwN.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix data races between :c:func:`PyDict_Watch` / :c:func:`PyDict_Unwatch` -and concurrent dict mutation in the :term:`free-threaded build`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-10-40-57.gh-issue-148418.ggA1LZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-10-40-57.gh-issue-148418.ggA1LZ.rst deleted file mode 100644 index 793858be7814c14..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-10-40-57.gh-issue-148418.ggA1LZ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a possible reference leak in a corrupted ``TYPE_CODE`` marshal stream. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-17-27-28.gh-issue-148390.MAhw7F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-17-27-28.gh-issue-148390.MAhw7F.rst deleted file mode 100644 index 881964673307ccc..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-12-17-27-28.gh-issue-148390.MAhw7F.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix an undefined behavior in :class:`memoryview` when using the native -boolean format (``?``) in :meth:`~memoryview.cast`. Previously, on some -common platforms, calling ``memoryview(b).cast("?").tolist()`` incorrectly -returned ``[False]`` instead of ``[True]`` for any even byte *b*. -Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-16-52-33.gh-issue-148515.09xulC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-16-52-33.gh-issue-148515.09xulC.rst deleted file mode 100644 index 53d91c8198f3d0f..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-16-52-33.gh-issue-148515.09xulC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug in the JIT optimizer reading operands for uops with multiple -caches. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst deleted file mode 100644 index 282b99176642bc3..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-23-21-45.gh-issue-145239.pL8qRt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Unary plus is now accepted in :keyword:`match` literal patterns, mirroring -the existing support for unary minus. -Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-15-48-04.gh-issue-148450.2MEVqH.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-15-48-04.gh-issue-148450.2MEVqH.rst new file mode 100644 index 000000000000000..2a7d0d9bb3a7f7e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-15-15-48-04.gh-issue-148450.2MEVqH.rst @@ -0,0 +1 @@ +Fix ``abc.register()`` so it invalidates type version tags for registered classes. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-11-30-00.gh-issue-142516.GcGen315.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-11-30-00.gh-issue-142516.GcGen315.rst deleted file mode 100644 index 5abb2485c20f12d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-11-30-00.gh-issue-142516.GcGen315.rst +++ /dev/null @@ -1 +0,0 @@ -Forward-port the generational cycle garbage collector to the default 3.15 build, replacing the incremental collector while leaving the free-threaded collector unchanged. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst deleted file mode 100644 index d3242235c6059bc..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-17-20-37-02.gh-issue-148653.nbbHMh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Forbid :mod:`marshalling <marshal>` recursive code objects, :class:`slice` -and :class:`frozendict` objects which cannot be correctly unmarshalled. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-18-16-41-04.gh-issue-148571.Q6WB3A.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-18-16-41-04.gh-issue-148571.Q6WB3A.rst deleted file mode 100644 index 70eeada34320ac7..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-18-16-41-04.gh-issue-148571.Q6WB3A.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash in the JIT optimizer when specialized opcode families inherited incompatible recorded operand layouts. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst deleted file mode 100644 index 3d9b4faa6ca4437..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-06-43-32.gh-issue-148829.GtIrYO.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :class:`sentinel`, implementing :pep:`661`. PEP by Tal Einat; patch by -Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst deleted file mode 100644 index 392becaffb73cf7..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-14-36-44.gh-issue-148820.XhOGhA.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix a race in :c:type:`!_PyRawMutex` on the free-threaded build where a -``Py_PARK_INTR`` return from ``_PySemaphore_Wait`` could let the waiter -destroy its semaphore before the unlocking thread's -``_PySemaphore_Wakeup`` completed, causing a fatal ``ReleaseSemaphore`` -error. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-15-07-03.gh-issue-148825.AbJzmZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-15-07-03.gh-issue-148825.AbJzmZ.rst new file mode 100644 index 000000000000000..5cf727b5d91fad1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-15-07-03.gh-issue-148825.AbJzmZ.rst @@ -0,0 +1 @@ +Fix build error if specialization is disabled. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-19-29-29.gh-issue-148850.MSH0J_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-19-29-29.gh-issue-148850.MSH0J_.rst deleted file mode 100644 index 324d1610310158d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-21-19-29-29.gh-issue-148850.MSH0J_.rst +++ /dev/null @@ -1 +0,0 @@ -Fix the memory sanitizer false positive in :func:`os.getrandom`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst deleted file mode 100644 index 54c04bbc28d4163..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-22-14-55-18.gh-issue-113956.0VEXd6.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a data race in :func:`sys.intern` in the free-threaded build when -interning a string owned by another thread. An interned copy owned by the -current thread is used instead when it is not safe to immortalize the -original. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-05-59-17.gh-issue-83065.f0UPNE.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-05-59-17.gh-issue-83065.f0UPNE.rst deleted file mode 100644 index 81bfa45c069fd45..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-05-59-17.gh-issue-83065.f0UPNE.rst +++ /dev/null @@ -1,7 +0,0 @@ -Fix a deadlock that could occur when one thread is importing a submodule -(for example ``import pkg.sub.mod``) while another thread is importing one -of its parent packages (for example ``import pkg.sub``) and that parent's -``__init__.py`` itself imports the submodule. The import system now -acquires module locks in hierarchical (parent-before-child) order so the -two threads serialise instead of raising -``_DeadlockError``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-21-19-21.gh-issue-149049.98u2Ib.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-21-19-21.gh-issue-149049.98u2Ib.rst deleted file mode 100644 index 4c8f7e08a44859d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-28-21-19-21.gh-issue-149049.98u2Ib.rst +++ /dev/null @@ -1 +0,0 @@ -Fix stack underflow for ``BINARY_OP`` in tier 2. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst deleted file mode 100644 index f34b6ea857a06cb..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-14-06-00.gh-issue-149122.P8k2Lm.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a crash in optimized calls to :func:`all`, :func:`any`, :func:`tuple`, -:func:`list`, and :func:`set` with an async generator expression argument -(for example, ``tuple(await x for x in y)``). These calls now correctly raise -``TypeError`` instead of crashing. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-30-01-35-09.gh-issue-149171.meXWpl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-30-01-35-09.gh-issue-149171.meXWpl.rst deleted file mode 100644 index f8320dd4c6b3c2d..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-30-01-35-09.gh-issue-149171.meXWpl.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow assignment to the ``__module__`` attribute of -:class:`typing.TypeAliasType` instances. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-03-10-24-50.gh-issue-149243.Zh1q9_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-03-10-24-50.gh-issue-149243.Zh1q9_.rst deleted file mode 100644 index 5c1956fe3983645..000000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-03-10-24-50.gh-issue-149243.Zh1q9_.rst +++ /dev/null @@ -1 +0,0 @@ -Check for recursion limits in ``CALL_ALLOC_AND_ENTER_INIT`` opcode. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-05-12-00-00.gh-issue-149321.remove-lazy-imports-none.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-05-12-00-00.gh-issue-149321.remove-lazy-imports-none.rst new file mode 100644 index 000000000000000..44e96ce7be0bfb4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-05-12-00-00.gh-issue-149321.remove-lazy-imports-none.rst @@ -0,0 +1 @@ +Do not support ``none`` as a lazy imports mode. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst new file mode 100644 index 000000000000000..4cd0a148df3c704 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-07-03-18-59.gh-issue-149459.5fhAqP.rst @@ -0,0 +1 @@ +Fix a crash in the JIT optimizer when a specialized ``LOAD_SPECIAL`` guard deoptimized after inserting the synthetic ``NULL`` stack entry. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst new file mode 100644 index 000000000000000..3063f1a3c0e6d3e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-09-15-22-32.gh-issue-144957.u1F2aQ.rst @@ -0,0 +1,2 @@ +Fix lazy ``from`` imports of module attributes provided by module-level +``__getattr__``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst new file mode 100644 index 000000000000000..815a084db69d8d8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-07-42-36.gh-issue-149642.6ZksML.rst @@ -0,0 +1,2 @@ +Allow imports inside ``exec()`` calls within functions under +``PYTHON_LAZY_IMPORTS=all``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-16-43-50.gh-issue-148829.gscS14.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-16-43-50.gh-issue-148829.gscS14.rst new file mode 100644 index 000000000000000..3f9b1ccb518787d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-10-16-43-50.gh-issue-148829.gscS14.rst @@ -0,0 +1,2 @@ +:class:`sentinel` objects now support a ``repr=`` argument and their +:attr:`~sentinel.__module__` attribute is writable. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst new file mode 100644 index 000000000000000..96f407cf5ad25a1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-14-48-56.gh-issue-149676.6aTrw1.rst @@ -0,0 +1 @@ +Fix ``frozendict | frozendict`` hash. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-18-20-41.gh-issue-148871.AeCbq7.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-18-20-41.gh-issue-148871.AeCbq7.rst new file mode 100644 index 000000000000000..506e369f553e506 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-18-20-41.gh-issue-148871.AeCbq7.rst @@ -0,0 +1,3 @@ +The empty tuple ``()`` is now loaded via :opcode:`LOAD_COMMON_CONSTANT` +instead of :opcode:`LOAD_CONST`, removing it from per-code-object +:attr:`~codeobject.co_consts` tuples. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-12-49-30.gh-issue-127727.hNmj9G.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-12-49-30.gh-issue-127727.hNmj9G.rst new file mode 100644 index 000000000000000..474ae434f6885ea --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-12-49-30.gh-issue-127727.hNmj9G.rst @@ -0,0 +1,3 @@ +Warn when running a virtual environment created for a different minor Python +version than the current interpreter, and suggest using ``python -m venv +--upgrade``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst new file mode 100644 index 000000000000000..3e9d930bf1de894 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-12-16-47-23.gh-issue-139808.iIs7_E.rst @@ -0,0 +1,2 @@ +Add branch protections for AArch64 (BTI/PAC) in assembly code used by +:option:`-X perf_jit <-X>` (Linux perf profiler integration). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-06-54-41.gh-issue-149738.4BLFoH.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-06-54-41.gh-issue-149738.4BLFoH.rst new file mode 100644 index 000000000000000..e62b681d716650b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-06-54-41.gh-issue-149738.4BLFoH.rst @@ -0,0 +1,2 @@ +:mod:`sqlite3`: Disallow removing ``row_factory`` and ``text_factory`` attributes +of a connection to prevent a crash on a query. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-21-26-26.gh-issue-149805.IG6cza.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-21-26-26.gh-issue-149805.IG6cza.rst new file mode 100644 index 000000000000000..02d050840ee1f9b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-13-21-26-26.gh-issue-149805.IG6cza.rst @@ -0,0 +1,2 @@ +Fix a :exc:`SystemError` when compiling a compiling ``__classdict__`` class +annotation. Found by OSS-Fuzz in :oss-fuzz:`512907042`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst new file mode 100644 index 000000000000000..a94c737e73619d8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-14-19-41-03.gh-issue-149807.IwGaCo.rst @@ -0,0 +1,2 @@ +Fix ``hash(frozendict)``: compute the hash of each ``(key, value)`` pair +correctly. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst new file mode 100644 index 000000000000000..016c17dd66b19ea --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-15-11-31-57.gh-issue-149816.ugN2rx.rst @@ -0,0 +1 @@ +Fix a race condition in :class:`memoryview` with free-threading. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst new file mode 100644 index 000000000000000..d35f0857a1aefe8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-16-11-03-54.gh-issue-149816.X_gqMT.rst @@ -0,0 +1 @@ +Fix a race condition in ``_PyBytes_FromList`` in free-threading mode. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-17-12-00-00.gh-issue-138325.h2XSfe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-17-12-00-00.gh-issue-138325.h2XSfe.rst new file mode 100644 index 000000000000000..0eb528c68d8d6f0 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-17-12-00-00.gh-issue-138325.h2XSfe.rst @@ -0,0 +1,3 @@ +Speed up converting a list to a tuple in the ``tuple(genexpr)`` fast path +and in starred tuple displays (e.g. ``(*a, *b)``) by stealing the list's +items into the tuple instead of copying them. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst new file mode 100644 index 000000000000000..8d3b29d69cc8578 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-13-47-17.gh-issue-149590.IPBeQx.rst @@ -0,0 +1 @@ +Fix crash when faulthandler is imported more than once. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst new file mode 100644 index 000000000000000..18a4fbd9dadd60f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-16-54-54.gh-issue-150042.LSr5W8.rst @@ -0,0 +1 @@ +Fix refleak in queue.SimpleQueue.put if memory allocation fails. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-17-16-51.gh-issue-150027.sJgLvd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-17-16-51.gh-issue-150027.sJgLvd.rst new file mode 100644 index 000000000000000..66446105da3fba2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-17-16-51.gh-issue-150027.sJgLvd.rst @@ -0,0 +1,2 @@ +Improve performance of :class:`frozenset` objects by avoiding copies during +construction. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst new file mode 100644 index 000000000000000..d623dbdd75f0c8e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-18-18-36-28.gh-issue-148587.-RD3z5.rst @@ -0,0 +1 @@ +:py:attr:`sys.lazy_modules` is now a set instead of a dict as initially spelled out in PEP 810. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst new file mode 100644 index 000000000000000..f373f0bee7023ef --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-20-13-06-17.gh-issue-150146.i5m_SL.rst @@ -0,0 +1,5 @@ +Fix a crash on a complex type variable substitution. + +``from typing import TypeVar; memoryview[TypeVar("")][*typing.Mapping[..., +...]]`` used to fail due to missing ``NULL`` check on ``_unpack_args`` C +function call. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-21-21-04-43.gh-issue-150208.VOkxRG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-21-21-04-43.gh-issue-150208.VOkxRG.rst new file mode 100644 index 000000000000000..e9bc1d9e976523b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-21-21-04-43.gh-issue-150208.VOkxRG.rst @@ -0,0 +1,2 @@ +Avoid double-quoting string values from ``pyconfig.h`` in ``sysconfigdata`` +variables. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst new file mode 100644 index 000000000000000..a13f249e48cc021 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-17-09-28.gh-issue-150107.GD72-D.rst @@ -0,0 +1,3 @@ +:mod:`asyncio`: ``sendfile()`` and ``sock_sendfile()`` event loop methods +now call ``file.seek(offset)`` if *file* has a ``seek()`` method, +even if *offset* is ``0`` (default value). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst new file mode 100644 index 000000000000000..12fbffcd170684c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-22-21-52-38.gh-issue-150207.l2BUtI.rst @@ -0,0 +1 @@ +Fix a crash when a memory allocation fails during tokenizer initialization. A proper :exc:`MemoryError` is now raised instead. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-23-22-08-01.gh-issue-149449.2lhQFF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-23-22-08-01.gh-issue-149449.2lhQFF.rst new file mode 100644 index 000000000000000..7d11442468d2077 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-23-22-08-01.gh-issue-149449.2lhQFF.rst @@ -0,0 +1,3 @@ +Fix a use-after-free crash when the :mod:`unicodedata` module was removed +from :data:`sys.modules` and garbage-collected between calls that decode +``\N{...}`` escapes or use the ``namereplace`` codec error handler. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-14-45-00.gh-issue-149156.NP73rB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-14-45-00.gh-issue-149156.NP73rB.rst new file mode 100644 index 000000000000000..2cb091e2b162f6a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-14-45-00.gh-issue-149156.NP73rB.rst @@ -0,0 +1,3 @@ +Fix an intermittent crash after :func:`os.fork` when perf trampoline +profiling is enabled and the child returns through trampoline frames +inherited from the parent process. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-22-46-49.gh-issue-148613.PLpmyd.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-22-46-49.gh-issue-148613.PLpmyd.rst new file mode 100644 index 000000000000000..71a701bf3eb3551 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-24-22-46-49.gh-issue-148613.PLpmyd.rst @@ -0,0 +1,2 @@ +Fix a data race in the free-threaded build between :func:`gc.set_threshold` +and garbage collection scheduling during object allocation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-25-16-00-22.gh-issue-150374.Emu6d8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-25-16-00-22.gh-issue-150374.Emu6d8.rst new file mode 100644 index 000000000000000..7189ca186d2b7e0 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-25-16-00-22.gh-issue-150374.Emu6d8.rst @@ -0,0 +1 @@ +Fix double release of the import lock on lazy import reification errors. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-26-00-06-30.gh-issue-150411.u-d-_5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-26-00-06-30.gh-issue-150411.u-d-_5.rst new file mode 100644 index 000000000000000..5b19a4fff5ddc73 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-26-00-06-30.gh-issue-150411.u-d-_5.rst @@ -0,0 +1,2 @@ +Fix a data race in the free-threaded build when :func:`gc.get_count` reads +the young generation allocation count while another thread updates it. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-17-51-11.gh-issue-144774.jPcixe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-17-51-11.gh-issue-144774.jPcixe.rst new file mode 100644 index 000000000000000..3dd117e7d3c3502 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-17-51-11.gh-issue-144774.jPcixe.rst @@ -0,0 +1 @@ +Fix data race in :class:`BaseException` when an exception is copied while being mutated. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-30-20-19-35.gh-issue-150633.XkNul0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-30-20-19-35.gh-issue-150633.XkNul0.rst new file mode 100644 index 000000000000000..c397ad61f086c1b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-30-20-19-35.gh-issue-150633.XkNul0.rst @@ -0,0 +1,3 @@ +Fix the frozen importer accepting module names with embedded null bytes, which +caused it to bypass the :data:`sys.modules` cache and create duplicate module +objects. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-00-00.gh-issue-150700.W8CzVR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-00-00.gh-issue-150700.W8CzVR.rst new file mode 100644 index 000000000000000..e7734034ff5c814 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-00-00.gh-issue-150700.W8CzVR.rst @@ -0,0 +1,3 @@ +Fix a :exc:`SystemError` when compiling a class-scope comprehension containing +a ``lambda`` that references ``__class__``, ``__classdict__``, or +``__conditional_annotations__``. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-21-01.gh-issue-150723.Hb3JDG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-21-01.gh-issue-150723.Hb3JDG.rst new file mode 100644 index 000000000000000..1920c8cdfce4f4c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-21-01.gh-issue-150723.Hb3JDG.rst @@ -0,0 +1,4 @@ +Fix malformed perf jitdump thread ids on macOS. The ``thread_id`` field of the +``JR_CODE_LOAD`` record was written as a 64-bit value instead of the 32-bit +value required by the jitdump format, which shifted every following field and +prevented profilers from resolving Python frames. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-24-12.gh-issue-150723.WlcL_-.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-24-12.gh-issue-150723.WlcL_-.rst new file mode 100644 index 000000000000000..78c896b669c2393 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-01-19-24-12.gh-issue-150723.WlcL_-.rst @@ -0,0 +1,4 @@ +Fix perf jitdump timestamps on macOS. Events were stamped using +``CLOCK_MONOTONIC``, but macOS profilers timestamp their samples with +``mach_absolute_time()``. The mismatch prevented the JIT code mappings from +lining up with the samples, so no Python frame could be resolved. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-03-16-27-00.gh-issue-150858.j2dSkD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-03-16-27-00.gh-issue-150858.j2dSkD.rst new file mode 100644 index 000000000000000..dcb932bf320629a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-03-16-27-00.gh-issue-150858.j2dSkD.rst @@ -0,0 +1 @@ +Fix a data race while changing ``__qualname__`` of a type concurrently on free-threaded builds. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-22-52-41.gh-issue-150988.fDKfMJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-22-52-41.gh-issue-150988.fDKfMJ.rst new file mode 100644 index 000000000000000..6fb70a1ce2685c8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-22-52-41.gh-issue-150988.fDKfMJ.rst @@ -0,0 +1,2 @@ +Fix a reference leak in :exc:`OSError` when attributes are set before +``super().__init__()``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-06-08-20-00.gh-issue-150942.Jk9pQr.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-06-08-20-00.gh-issue-150942.Jk9pQr.rst new file mode 100644 index 000000000000000..9777b8932271404 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-06-08-20-00.gh-issue-150942.Jk9pQr.rst @@ -0,0 +1,3 @@ +Speed up frame local variable item collection by appending result pairs to the +output list without an extra reference-count round-trip (using the internal +reference-stealing list append helper). Patch by Omkar Kabde. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-05-31-22.gh-issue-151065._o_31F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-05-31-22.gh-issue-151065._o_31F.rst new file mode 100644 index 000000000000000..e46c96ef784cc9e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-05-31-22.gh-issue-151065._o_31F.rst @@ -0,0 +1 @@ +Fix memory leak when using the :ref:`mimalloc memory allocator <mimalloc>`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-13-14-42.gh-issue-150902.-CWZ66.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-13-14-42.gh-issue-150902.-CWZ66.rst new file mode 100644 index 000000000000000..e3b7cd387b439fe --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-13-14-42.gh-issue-150902.-CWZ66.rst @@ -0,0 +1 @@ +Apply an existing optimization of PyCriticalSection (single mutex) to PyCriticalSection2: avoid acquiring the same locks that the current CS has already acquired. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-10-28-30.gh-issue-151126.DKa6Sl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-10-28-30.gh-issue-151126.DKa6Sl.rst new file mode 100644 index 000000000000000..67e2ce4044431f9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-10-28-30.gh-issue-151126.DKa6Sl.rst @@ -0,0 +1,5 @@ +Fix a crash, when there's no memory left on a device, +which happened in: code compilation, :mod:`!_interpchannels` module, +:func:`!_winapi.CreateProcess` function. + +Now these places raise proper :exc:`MemoryError` errors. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-12-24-35.gh-issue-151112.4RKCkD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-12-24-35.gh-issue-151112.4RKCkD.rst new file mode 100644 index 000000000000000..93ee5c8cf1914b4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-12-24-35.gh-issue-151112.4RKCkD.rst @@ -0,0 +1 @@ +Fix a crash in the compiler that could occur when running out of memory. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-19-58.gh-issue-151238.C9Wu4x.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-19-58.gh-issue-151238.C9Wu4x.rst new file mode 100644 index 000000000000000..fe7519fb4878952 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-19-58.gh-issue-151238.C9Wu4x.rst @@ -0,0 +1,2 @@ +Fix a crash when compiling a concatenated f-string or t-string if an error +occurs when processing one of it's parts. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-42-46.gh-issue-151253.7MMQ8P.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-42-46.gh-issue-151253.7MMQ8P.rst new file mode 100644 index 000000000000000..56d2f3b2633bb01 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-42-46.gh-issue-151253.7MMQ8P.rst @@ -0,0 +1,3 @@ +If ``import encodings`` (first import) fails at Python startup, dump the +Python path configuration to help users debugging their configuration. Patch +by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-11-16-03-23.gh-issue-151297.NGPkUM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-11-16-03-23.gh-issue-151297.NGPkUM.rst new file mode 100644 index 000000000000000..288d726e0f1004d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-11-16-03-23.gh-issue-151297.NGPkUM.rst @@ -0,0 +1 @@ +Fix an invalid pointer dereference that could occur when calling :c:func:`PyObject_Realloc` with a NULL pointer in :term:`free-threaded builds <free-threaded build>` or with :envvar:`PYTHONMALLOC` set to ``mimalloc``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-12-15-30-25.gh-issue-151218.5M_nv8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-12-15-30-25.gh-issue-151218.5M_nv8.rst new file mode 100644 index 000000000000000..46539efc373eb0d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-12-15-30-25.gh-issue-151218.5M_nv8.rst @@ -0,0 +1,3 @@ +:c:func:`PyConfig_Set` and :func:`sys.set_int_max_str_digits` now replace +:data:`sys.flags` (create a new object), instead of modifying :data:`sys.flags` +in-place. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst new file mode 100644 index 000000000000000..d76a9bc95278bcb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst @@ -0,0 +1,3 @@ +Fix direct execution of files with invalid source encodings to report the +underlying codec lookup or decoding error instead of the generic +``SyntaxError: encoding problem`` message. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-00-45-42.gh-issue-151510.HJ-kGn.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-00-45-42.gh-issue-151510.HJ-kGn.rst new file mode 100644 index 000000000000000..cfa5ee8d3839c1b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-00-45-42.gh-issue-151510.HJ-kGn.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`!__lazy_import__` when called without an explicit +``globals`` argument and without a current Python frame. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-17-23-37.gh-issue-151546.LhiaZz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-17-23-37.gh-issue-151546.LhiaZz.rst new file mode 100644 index 000000000000000..af1c23bd50355f2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-17-23-37.gh-issue-151546.LhiaZz.rst @@ -0,0 +1,3 @@ +Fix the stack limit check if Python is linked to musl (ex: Alpine Linux). +Use the stack size set by the linker to compute the stack limits. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-26-45.gh-issue-151126.Yc_6wC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-26-45.gh-issue-151126.Yc_6wC.rst new file mode 100644 index 000000000000000..29bb03d2d408598 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-26-45.gh-issue-151126.Yc_6wC.rst @@ -0,0 +1,2 @@ +Avoid possible crash in ``getpath.c`` where a device has no memory left. Now +it properly raises a :exc:`MemoryError`. Patch by Ivy Xu. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-46-07.gh-issue-151126.vhTL0T.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-46-07.gh-issue-151126.vhTL0T.rst new file mode 100644 index 000000000000000..6f2d230b1dcfc00 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-46-07.gh-issue-151126.vhTL0T.rst @@ -0,0 +1,2 @@ +Avoid possible crash in ``_winapi.c`` where a device has no memory left. Now +it properly raises a :exc:`MemoryError`. Patch by Ivy Xu. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-16-00-10.gh-issue-151126.tBqn6I.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-16-00-10.gh-issue-151126.tBqn6I.rst new file mode 100644 index 000000000000000..d495df43ede932c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-16-00-10.gh-issue-151126.tBqn6I.rst @@ -0,0 +1,3 @@ +Fix a crash when sharing :class:`memoryview` objects between interpreters +fails due to running out of memory. It now raises a proper +:exc:`MemoryError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-20-00-30-47.gh-issue-151722.RPMPIY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-20-00-30-47.gh-issue-151722.RPMPIY.rst new file mode 100644 index 000000000000000..57b5dee7458ede5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-20-00-30-47.gh-issue-151722.RPMPIY.rst @@ -0,0 +1,2 @@ +Defer GC tracking of :class:`frozendict` to end of construction. Patch by +Donghee Na. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-12-55-18.gh-issue-151665.82fmzx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-12-55-18.gh-issue-151665.82fmzx.rst new file mode 100644 index 000000000000000..d08a1220cbe5efc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-12-55-18.gh-issue-151665.82fmzx.rst @@ -0,0 +1,2 @@ +:func:`inspect.signature` now works on the lazy evaluators of type aliases +and type parameters instead of raising :exc:`ValueError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-16-00-00.gh-issue-151773.mN4kRt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-16-00-00.gh-issue-151773.mN4kRt.rst new file mode 100644 index 000000000000000..b4193c5e15ffef8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-16-00-00.gh-issue-151773.mN4kRt.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`contextvars.ContextVar.set` when memory allocation +fails. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-06-26-34.gh-issue-151905.FOLMYg.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-06-26-34.gh-issue-151905.FOLMYg.rst new file mode 100644 index 000000000000000..c71122df6b8580f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-06-26-34.gh-issue-151905.FOLMYg.rst @@ -0,0 +1 @@ +Fix OOM error handling in :c:func:`PyFrame_GetBack` to propagate exceptions instead of masking them as None. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-19-50-22.gh-issue-152020.DTKXjR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-19-50-22.gh-issue-152020.DTKXjR.rst new file mode 100644 index 000000000000000..93c716f7a6a1c86 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-19-50-22.gh-issue-152020.DTKXjR.rst @@ -0,0 +1,3 @@ +On the free-threaded build, :func:`asyncio.all_tasks` no longer loses +eager-started tasks when called from a thread other than the one running the +event loop. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-23-48-54.gh-issue-151763.Eu8pYQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-23-48-54.gh-issue-151763.Eu8pYQ.rst new file mode 100644 index 000000000000000..d4746e992f8779d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-23-48-54.gh-issue-151763.Eu8pYQ.rst @@ -0,0 +1 @@ +Fixes possible crash on :class:`types.CodeType` deallocation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-24-13-36-31.gh-issue-151722.lWKfE1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-24-13-36-31.gh-issue-151722.lWKfE1.rst new file mode 100644 index 000000000000000..db50b0058bdbdda --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-24-13-36-31.gh-issue-151722.lWKfE1.rst @@ -0,0 +1,3 @@ +:meth:`!frozendict.fromkeys` now only tracks the :class:`frozendict` in the +garbage collector once the dictionary is fully initialized. Patch by Donghee Na +and Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-05-49-13.gh-issue-152235.YU20T9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-05-49-13.gh-issue-152235.YU20T9.rst new file mode 100644 index 000000000000000..8d386ad458dfff1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-05-49-13.gh-issue-152235.YU20T9.rst @@ -0,0 +1,2 @@ +Defer GC tracking of a :class:`set` or :class:`frozenset` to the end of its +construction from iterable. Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Documentation/2023-09-16-23-42-27.gh-issue-109503.mZ-kdU.rst b/Misc/NEWS.d/next/Documentation/2023-09-16-23-42-27.gh-issue-109503.mZ-kdU.rst new file mode 100644 index 000000000000000..c3c6c57569c2ea5 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-09-16-23-42-27.gh-issue-109503.mZ-kdU.rst @@ -0,0 +1,3 @@ +Fix documentation for :func:`shutil.move` on usage of +:func:`os.rename` since nonatomic move might be used even if the files are +on the same filesystem. Patch by Fang Li diff --git a/Misc/NEWS.d/next/Documentation/2026-04-02-07-20-00.gh-issue-146646.GlobDoc1.rst b/Misc/NEWS.d/next/Documentation/2026-04-02-07-20-00.gh-issue-146646.GlobDoc1.rst deleted file mode 100644 index 4e89270442a33b1..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2026-04-02-07-20-00.gh-issue-146646.GlobDoc1.rst +++ /dev/null @@ -1,3 +0,0 @@ -Document that :func:`glob.glob`, :func:`glob.iglob`, -:meth:`pathlib.Path.glob`, and :meth:`pathlib.Path.rglob` silently suppress -:exc:`OSError` exceptions raised from scanning the filesystem. diff --git a/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst b/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst deleted file mode 100644 index 0fbe5a699ef0ad2..000000000000000 --- a/Misc/NEWS.d/next/Documentation/2026-04-17-02-28-55.gh-issue-148663.MHIbRB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Document that :class:`calendar.IllegalMonthError` is a subclass of both -:exc:`ValueError` and :exc:`IndexError` since Python 3.12. diff --git a/Misc/NEWS.d/next/Documentation/2026-05-23-17-27-41.gh-issue-150319.ol9tWK.rst b/Misc/NEWS.d/next/Documentation/2026-05-23-17-27-41.gh-issue-150319.ol9tWK.rst new file mode 100644 index 000000000000000..d56ccbce2fa325c --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-05-23-17-27-41.gh-issue-150319.ol9tWK.rst @@ -0,0 +1,2 @@ +Generic builtin and standard library types now document the meaning of their +type parameters. diff --git a/Misc/NEWS.d/next/Documentation/2026-06-17-12-00-00.gh-issue-86726.__bOgH.rst b/Misc/NEWS.d/next/Documentation/2026-06-17-12-00-00.gh-issue-86726.__bOgH.rst new file mode 100644 index 000000000000000..787b95dbf236279 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-06-17-12-00-00.gh-issue-86726.__bOgH.rst @@ -0,0 +1,4 @@ +Greatly expand the :mod:`tkinter` documentation to cover the full public API +of the package and its submodules. The descriptions are oriented towards +Python rather than Tcl/Tk, with corrected return types and +``versionadded``/``versionchanged`` information. diff --git a/Misc/NEWS.d/next/IDLE/2019-12-12-03-18-02.bpo-6699.1CqJFG.rst b/Misc/NEWS.d/next/IDLE/2019-12-12-03-18-02.bpo-6699.1CqJFG.rst new file mode 100644 index 000000000000000..e7fb9bf1b3bdf6a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2019-12-12-03-18-02.bpo-6699.1CqJFG.rst @@ -0,0 +1 @@ +Warn the user if a file will be overwritten when saving. diff --git a/Misc/NEWS.d/next/IDLE/2021-10-03-21-55-34.gh-issue-89520.etEExa.rst b/Misc/NEWS.d/next/IDLE/2021-10-03-21-55-34.gh-issue-89520.etEExa.rst deleted file mode 100644 index e8e181cac213589..000000000000000 --- a/Misc/NEWS.d/next/IDLE/2021-10-03-21-55-34.gh-issue-89520.etEExa.rst +++ /dev/null @@ -1,3 +0,0 @@ -Make IDLE extension configuration look at user config files, allowing -user-installed extensions to have settings and key bindings defined in -~/.idlerc. diff --git a/Misc/NEWS.d/next/IDLE/2025-10-05-19-33-39.gh-issue-139551.TX9BRc.rst b/Misc/NEWS.d/next/IDLE/2025-10-05-19-33-39.gh-issue-139551.TX9BRc.rst deleted file mode 100644 index 5ea1dfc9b5100d3..000000000000000 --- a/Misc/NEWS.d/next/IDLE/2025-10-05-19-33-39.gh-issue-139551.TX9BRc.rst +++ /dev/null @@ -1 +0,0 @@ -Support rendering :exc:`BaseExceptionGroup` in IDLE. diff --git a/Misc/NEWS.d/next/IDLE/2026-03-07-20-47-40.gh-issue-94523.dq7m2k.rst b/Misc/NEWS.d/next/IDLE/2026-03-07-20-47-40.gh-issue-94523.dq7m2k.rst deleted file mode 100644 index 831f6ac628768c7..000000000000000 --- a/Misc/NEWS.d/next/IDLE/2026-03-07-20-47-40.gh-issue-94523.dq7m2k.rst +++ /dev/null @@ -1,2 +0,0 @@ -Detect file if modified at local disk and prompt to ask refresh. Patch by -Shixian Li. diff --git a/Misc/NEWS.d/next/Library/2021-02-26-13-17-57.bpo-40469.yJHeQg.rst b/Misc/NEWS.d/next/Library/2021-02-26-13-17-57.bpo-40469.yJHeQg.rst new file mode 100644 index 000000000000000..239f856bb04c0ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-02-26-13-17-57.bpo-40469.yJHeQg.rst @@ -0,0 +1,5 @@ +:class:`~logging.handlers.TimedRotatingFileHandler` now uses the creation time +instead of the last modification time of an existing log file as the basis +for the first rotation after handler creation, if supported by the OS and file system. +This allows it to be used in short-running programs that start and end before +the rotation interval expires. diff --git a/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst b/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst new file mode 100644 index 000000000000000..80c38c03f8fe787 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-18-13-46-55.bpo-45509.Upwb60.rst @@ -0,0 +1 @@ +Gzip headers are now checked for corrupted NAME, COMMENT and HCRC fields. diff --git a/Misc/NEWS.d/next/Library/2022-04-04-17-58-05.bpo-47216.gPyPte.rst b/Misc/NEWS.d/next/Library/2022-04-04-17-58-05.bpo-47216.gPyPte.rst new file mode 100644 index 000000000000000..066e17c1c6e06aa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-04-04-17-58-05.bpo-47216.gPyPte.rst @@ -0,0 +1,2 @@ +Added *mtime* option to :func:`gzip.open`, which will be passed +to the constructor of :class:`~gzip.GzipFile`. diff --git a/Misc/NEWS.d/next/Library/2022-09-17-20-20-01.gh-issue-96894.t7my0A.rst b/Misc/NEWS.d/next/Library/2022-09-17-20-20-01.gh-issue-96894.t7my0A.rst deleted file mode 100644 index 546d47b8a2f28f6..000000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-17-20-20-01.gh-issue-96894.t7my0A.rst +++ /dev/null @@ -1,2 +0,0 @@ -Do not turn echo off for subsequent commands in batch activators -(``activate.bat`` and ``deactivate.bat``) of :mod:`venv`. diff --git a/Misc/NEWS.d/next/Library/2023-01-23-21-23-50.gh-issue-101267._f-cFH.rst b/Misc/NEWS.d/next/Library/2023-01-23-21-23-50.gh-issue-101267._f-cFH.rst new file mode 100644 index 000000000000000..901a3fb60ab5b9f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-23-21-23-50.gh-issue-101267._f-cFH.rst @@ -0,0 +1,7 @@ +When a worker process terminates unexpectedly, +:class:`concurrent.futures.ProcessPoolExecutor` now sets a separate +:exc:`~concurrent.futures.process.BrokenProcessPool` exception on each pending +future instead of sharing a single instance among them all. Sharing one +exception produced malformed tracebacks: each +:meth:`Future.result() <concurrent.futures.Future.result>` call re-raised the +same object, appending another copy of the traceback to it. diff --git a/Misc/NEWS.d/next/Library/2023-02-26-14-07-18.gh-issue-91099._QPbEL.rst b/Misc/NEWS.d/next/Library/2023-02-26-14-07-18.gh-issue-91099._QPbEL.rst new file mode 100644 index 000000000000000..d886e8ac6032a4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-26-14-07-18.gh-issue-91099._QPbEL.rst @@ -0,0 +1,2 @@ +:meth:`imaplib.IMAP4.login` now raises exceptions with :class:`str` instead of +:class:`bytes`. Patch by Florian Best. diff --git a/Misc/NEWS.d/next/Library/2023-09-08-13-10-32.gh-issue-83281.2Plpcj.rst b/Misc/NEWS.d/next/Library/2023-09-08-13-10-32.gh-issue-83281.2Plpcj.rst deleted file mode 100644 index cf2ae770bd1940f..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-09-08-13-10-32.gh-issue-83281.2Plpcj.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`email`: improve handling trailing garbage in address lists to avoid throwing -AttributeError in certain edge cases diff --git a/Misc/NEWS.d/next/Library/2023-12-25-19-14-07.gh-issue-113471.ZQMpbI.rst b/Misc/NEWS.d/next/Library/2023-12-25-19-14-07.gh-issue-113471.ZQMpbI.rst deleted file mode 100644 index 99ba9bd1820fc16..000000000000000 --- a/Misc/NEWS.d/next/Library/2023-12-25-19-14-07.gh-issue-113471.ZQMpbI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Allow :mod:`http.server` to set a default content-type when serving -files with an unknown or missing extension. diff --git a/Misc/NEWS.d/next/Library/2024-02-10-21-25-22.gh-issue-70039.6wvcAP.rst b/Misc/NEWS.d/next/Library/2024-02-10-21-25-22.gh-issue-70039.6wvcAP.rst deleted file mode 100644 index 8bb2cd188e89fa4..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-02-10-21-25-22.gh-issue-70039.6wvcAP.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed bug where :meth:`smtplib.SMTP.starttls` could fail if :meth:`smtplib.SMTP.connect` is called explicitly rather than implicitly. diff --git a/Misc/NEWS.d/next/Library/2024-03-17-22-16-53.gh-issue-115988.EshSDM.rst b/Misc/NEWS.d/next/Library/2024-03-17-22-16-53.gh-issue-115988.EshSDM.rst new file mode 100644 index 000000000000000..4efde6a48c26cc8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-17-22-16-53.gh-issue-115988.EshSDM.rst @@ -0,0 +1 @@ +:mod:`lzma` adds constants to support the newer BCJ filters for ARM64 and RISC-V. diff --git a/Misc/NEWS.d/next/Library/2024-06-18-04-08-37.gh-issue-120665.x7T1hV.rst b/Misc/NEWS.d/next/Library/2024-06-18-04-08-37.gh-issue-120665.x7T1hV.rst new file mode 100644 index 000000000000000..27e93988ed11efb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-18-04-08-37.gh-issue-120665.x7T1hV.rst @@ -0,0 +1 @@ +Fixed an issue where ``unittest`` loaders would load and instantiate :class:`unittest.TestCase`-derived subclasses that are also abstract base classes, which can't be instantiated. diff --git a/Misc/NEWS.d/next/Library/2024-07-02-20-57-43.gh-issue-121109.Tp6R2s.rst b/Misc/NEWS.d/next/Library/2024-07-02-20-57-43.gh-issue-121109.Tp6R2s.rst new file mode 100644 index 000000000000000..eca6014e4a0aed1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-02-20-57-43.gh-issue-121109.Tp6R2s.rst @@ -0,0 +1,2 @@ +Fix :mod:`tarfile` performance issue when reading archives in streaming mode +(e.g. ``r|*``). diff --git a/Misc/NEWS.d/next/Library/2024-07-30-19-19-33.gh-issue-81074.YAeWNf.rst b/Misc/NEWS.d/next/Library/2024-07-30-19-19-33.gh-issue-81074.YAeWNf.rst deleted file mode 100644 index 87de4fade14dfba..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-07-30-19-19-33.gh-issue-81074.YAeWNf.rst +++ /dev/null @@ -1,8 +0,0 @@ -The :mod:`email` module no longer treats email addresses with non-ASCII -characters as defects when parsing a Unicode string or in the ``addr_spec`` -parameter to :class:`email.headerregistry.Address`. :rfc:`5322` permits such -addresses, and they were already supported when parsing bytes and in the Address -``username`` parameter. - -The (undocumented) :exc:`!email.errors.NonASCIILocalPartDefect` is no longer -used and should be considered deprecated. diff --git a/Misc/NEWS.d/next/Library/2024-07-31-17-22-10.gh-issue-83938.TtUa-c.rst b/Misc/NEWS.d/next/Library/2024-07-31-17-22-10.gh-issue-83938.TtUa-c.rst deleted file mode 100644 index 7082c72f685b057..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-07-31-17-22-10.gh-issue-83938.TtUa-c.rst +++ /dev/null @@ -1,8 +0,0 @@ -The :mod:`email` module no longer incorrectly uses :rfc:`2047` encoding for -a mailbox with non-ASCII characters in its domain. Under a policy with -:attr:`~email.policy.EmailPolicy.utf8` set ``False``, attempting to serialize -such a message will now raise an :exc:`~email.errors.HeaderWriteError`. -Either apply an appropriate IDNA encoding to convert the domain to ASCII before -serialization, or use :data:`email.policy.SMTPUTF8` (or another policy with -``utf8=True``) to correctly pass through the internationalized domain name -as Unicode characters. diff --git a/Misc/NEWS.d/next/Library/2024-07-31-17-23-06.gh-issue-122476.TtUa-c.rst b/Misc/NEWS.d/next/Library/2024-07-31-17-23-06.gh-issue-122476.TtUa-c.rst deleted file mode 100644 index 29c076d3a746c60..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-07-31-17-23-06.gh-issue-122476.TtUa-c.rst +++ /dev/null @@ -1,7 +0,0 @@ -The :mod:`email` module no longer incorrectly uses :rfc:`2047` encoding for -a mailbox with non-ASCII characters in its local-part. Under a policy with -:attr:`~email.policy.EmailPolicy.utf8` set ``False``, attempting to serialize -such a message will now raise an :exc:`~email.errors.HeaderWriteError`. -There is no valid 7-bit encoding for an internationalized local-part. Use -:data:`email.policy.SMTPUTF8` (or another policy with ``utf8=True``) to -correctly pass through the local-part as Unicode characters. diff --git a/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst b/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst deleted file mode 100644 index d7204c289369a7b..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-09-09-12-48-37.gh-issue-123853.e-zFxb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Update the table of Windows language code identifiers (LCIDs) used by -:func:`locale.getdefaultlocale` on Windows to protocol version 16.0 -(2024-04-23). diff --git a/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst b/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst new file mode 100644 index 000000000000000..d5af322d68d309a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-02-02-02-31.gh-issue-107398.uUtA6Q.rst @@ -0,0 +1 @@ +Fix :mod:`tarfile` stream mode exception when process the file with the gzip extra field. diff --git a/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst b/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst deleted file mode 100644 index 0e0280c9b6b0e74..000000000000000 --- a/Misc/NEWS.d/next/Library/2024-11-24-07-18-40.gh-issue-108951.jyKygP.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`asyncio`: Add :meth:`TaskGroup.cancel <asyncio.TaskGroup.cancel>` which cancels -unfinished tasks and exits the group without raising :exc:`asyncio.CancelledError`. diff --git a/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst b/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst new file mode 100644 index 000000000000000..b08b1886cff9cf6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-01-13-36-02.gh-issue-128110.9wx_G0.rst @@ -0,0 +1,5 @@ +Fix bug in the parsing of :mod:`email` address headers that could result in +extraneous spaces in the decoded text when using a modern email policy. +Space between pairs of adjacent :rfc:`2047` encoded-words is now ignored, per +section 6.2 (and consistent with existing parsing of unstructured +headers like *Subject*). diff --git a/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst b/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst deleted file mode 100644 index 9cc1d5a389c0854..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-04-17-15-26-35.gh-issue-132631.IDFZfb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix "I/O operation on closed file" when parsing JSON Lines file with -:mod:`JSON CLI <json.tool>`. diff --git a/Misc/NEWS.d/next/Library/2025-05-01-16-44-16.gh-issue-72902.19qwJW.rst b/Misc/NEWS.d/next/Library/2025-05-01-16-44-16.gh-issue-72902.19qwJW.rst new file mode 100644 index 000000000000000..2b7d8e448408943 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-01-16-44-16.gh-issue-72902.19qwJW.rst @@ -0,0 +1,3 @@ +Optimize (~x1.4 speedup) :meth:`fractions.Fraction.from_decimal` and +:meth:`fractions.Fraction.from_float` for :class:`~decimal.Decimal` and +:class:`float` inputs, respectively. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2025-05-16-01-43-58.gh-issue-133956.5kWDYd.rst b/Misc/NEWS.d/next/Library/2025-05-16-01-43-58.gh-issue-133956.5kWDYd.rst deleted file mode 100644 index 5923e12d55964c4..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-05-16-01-43-58.gh-issue-133956.5kWDYd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix bug where :func:`@dataclass <dataclasses.dataclass>` -wouldn't detect ``ClassVar`` fields -if ``ClassVar`` was re-exported from a module -other than :mod:`typing`. diff --git a/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst b/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst new file mode 100644 index 000000000000000..77d92628beefacd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-20-29-35.gh-issue-133998.KmElUw.rst @@ -0,0 +1,5 @@ +Fix :exc:`struct.error` exception when creating a file with +:class:`gzip.GzipFile` or compressing data with :func:`gzip.compress` +if the system time is outside the range 00:00:00 UTC, January 1, 1970 +through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* +argument is outside the range ``0`` to ``2**32-1``. diff --git a/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst b/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst new file mode 100644 index 000000000000000..bf552fee814acbf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-21-08-25.gh-issue-134261.ravGYm.rst @@ -0,0 +1 @@ +zip: On reproducible builds, ZipFile uses UTC instead of the local time when writing file datetimes to avoid underflows. diff --git a/Misc/NEWS.d/next/Library/2025-05-23-10-28-51.gh-issue-134551.0rnq0X.rst b/Misc/NEWS.d/next/Library/2025-05-23-10-28-51.gh-issue-134551.0rnq0X.rst deleted file mode 100644 index 94e0c1e0b225d64..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-05-23-10-28-51.gh-issue-134551.0rnq0X.rst +++ /dev/null @@ -1 +0,0 @@ -Add t-strings support to pprint functions diff --git a/Misc/NEWS.d/next/Library/2025-05-24-11-17-34.gh-issue-51067.yHOgfy.rst b/Misc/NEWS.d/next/Library/2025-05-24-11-17-34.gh-issue-51067.yHOgfy.rst new file mode 100644 index 000000000000000..204213c74de5dfd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-24-11-17-34.gh-issue-51067.yHOgfy.rst @@ -0,0 +1 @@ +Add :meth:`~zipfile.ZipFile.remove` and :meth:`~zipfile.ZipFile.repack` to :class:`~zipfile.ZipFile`. diff --git a/Misc/NEWS.d/next/Library/2025-06-02-22-23-38.gh-issue-135056.yz3dSs.rst b/Misc/NEWS.d/next/Library/2025-06-02-22-23-38.gh-issue-135056.yz3dSs.rst deleted file mode 100644 index 754df083ab10638..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-02-22-23-38.gh-issue-135056.yz3dSs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a ``-H`` or ``--header`` CLI option to :program:`python -m http.server`. Contributed by -Anton I. Sipos. diff --git a/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst b/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst deleted file mode 100644 index ab3855582c77ee9..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-22-16-29-10.gh-issue-135528.Rt_QhR.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`http.cookiejar`: add "tv", "or", "nom", "sch", and "web" to the default list of supported country code second-level domains. diff --git a/Misc/NEWS.d/next/Library/2025-07-02-17-01-17.gh-issue-125862.WgFYj3.rst b/Misc/NEWS.d/next/Library/2025-07-02-17-01-17.gh-issue-125862.WgFYj3.rst deleted file mode 100644 index 1ccc91d55ec3ad7..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-02-17-01-17.gh-issue-125862.WgFYj3.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :func:`contextlib.contextmanager` and -:func:`contextlib.asynccontextmanager` decorators now work correctly with -generators, coroutine functions, and async generators when the wrapped -callables are used as decorators. diff --git a/Misc/NEWS.d/next/Library/2025-08-24-15-09-30.gh-issue-75707.GOWZrC.rst b/Misc/NEWS.d/next/Library/2025-08-24-15-09-30.gh-issue-75707.GOWZrC.rst deleted file mode 100644 index b2ff8a0cdf6f723..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-08-24-15-09-30.gh-issue-75707.GOWZrC.rst +++ /dev/null @@ -1 +0,0 @@ -Add optional ``mtime`` argument to :func:`tarfile.open`, for setting the ``mtime`` header field in ``.tar.gz`` archives. diff --git a/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst b/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst new file mode 100644 index 000000000000000..9c32671173e0ad2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-08-30-07-44-30.gh-issue-86533.pathlib.rst @@ -0,0 +1,4 @@ +The :func:`os.makedirs` function and :meth:`pathlib.Path.mkdir` method now have +a *parent_mode* parameter to specify the mode for intermediate directories when +creating parent directories. This allows one to match the behavior from Python +3.6 and earlier for :func:`os.makedirs`. diff --git a/Misc/NEWS.d/next/Library/2025-10-09-12-13-29.gh-issue-139819.YxUDyH.rst b/Misc/NEWS.d/next/Library/2025-10-09-12-13-29.gh-issue-139819.YxUDyH.rst new file mode 100644 index 000000000000000..ab3a8c6ed16ce4a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-09-12-13-29.gh-issue-139819.YxUDyH.rst @@ -0,0 +1,3 @@ +:mod:`rlcompleter`: Avoid suggesting attributes that are not accessible on +instances (e.g., Enum members showing ``__name__``). Patch by Peter +(ttw225). diff --git a/Misc/NEWS.d/next/Library/2025-10-12-08-46-26.gh-issue-139398.hESOMl.rst b/Misc/NEWS.d/next/Library/2025-10-12-08-46-26.gh-issue-139398.hESOMl.rst new file mode 100644 index 000000000000000..201b682d177c010 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-12-08-46-26.gh-issue-139398.hESOMl.rst @@ -0,0 +1,2 @@ +Add supported ``_sunder_`` names to the :func:`dir` method of the :mod:`enum` module +to support them in :term:`REPL` autocompletion. diff --git a/Misc/NEWS.d/next/Library/2025-10-18-12-13-39.gh-issue-140287.49iU-4.rst b/Misc/NEWS.d/next/Library/2025-10-18-12-13-39.gh-issue-140287.49iU-4.rst deleted file mode 100644 index 09643956d980938..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-10-18-12-13-39.gh-issue-140287.49iU-4.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :mod:`asyncio` REPL now handles exceptions when executing :envvar:`PYTHONSTARTUP` scripts. -Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2025-10-20-09-12-18.gh-issue-140344.WjbYg-.rst b/Misc/NEWS.d/next/Library/2025-10-20-09-12-18.gh-issue-140344.WjbYg-.rst new file mode 100644 index 000000000000000..ca3046941c6c93f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-20-09-12-18.gh-issue-140344.WjbYg-.rst @@ -0,0 +1,11 @@ +The classes ``ast.slice``, ``ast.ExtSlice``, ``ast.Index``, ``ast.Suite``, ``ast.AugLoad``, +``ast.AugStore``, and ``ast.Param``, deprecated since Python 3.9, now issue +deprecation warnings on use. They are now scheduled for removal in Python 3.21. + +The ``dims`` property of ``ast.Tuple`` objects, deprecated since Python 3.9, +now issues a deprecation warning when accessed. The property is scheduled for +removal in Python 3.21. Use the (non-deprecated) ``elts`` property of +``ast.Tuple`` objects instead. + +The deprecated global names are also no longer imported by +``from ast import *``. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-22-24-13.gh-issue-140924.NQVbR_.rst b/Misc/NEWS.d/next/Library/2025-11-02-22-24-13.gh-issue-140924.NQVbR_.rst new file mode 100644 index 000000000000000..91c30c95469f089 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-22-24-13.gh-issue-140924.NQVbR_.rst @@ -0,0 +1,3 @@ +Add :func:`locale.localize`, :func:`locale.delocalize` +and platform-specific locale constants +from the :mod:`!_locale` module to ``locale.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-12-06-08-48-26.gh-issue-141449.hQvNW_.rst b/Misc/NEWS.d/next/Library/2025-12-06-08-48-26.gh-issue-141449.hQvNW_.rst deleted file mode 100644 index 4e94c3c80d780ff..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-12-06-08-48-26.gh-issue-141449.hQvNW_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve tests and documentation for non-function callables as -:term:`annotate functions <annotate function>`. diff --git a/Misc/NEWS.d/next/Library/2025-12-10-15-15-09.gh-issue-130273.iCfiY5.rst b/Misc/NEWS.d/next/Library/2025-12-10-15-15-09.gh-issue-130273.iCfiY5.rst deleted file mode 100644 index 2e0695334fd71ec..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-12-10-15-15-09.gh-issue-130273.iCfiY5.rst +++ /dev/null @@ -1 +0,0 @@ -Fix traceback color output with Unicode characters. diff --git a/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst b/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst deleted file mode 100644 index 95aa41e922684fc..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-12-17-02-55-03.gh-issue-108411.up7MAc.rst +++ /dev/null @@ -1,2 +0,0 @@ -``typing.IO`` and ``typing.BinaryIO`` method arguments are now -positional-only. diff --git a/Misc/NEWS.d/next/Library/2025-12-17-04-10-35.gh-issue-142831.ee3t4L.rst b/Misc/NEWS.d/next/Library/2025-12-17-04-10-35.gh-issue-142831.ee3t4L.rst deleted file mode 100644 index 5fa3cd2727a9e57..000000000000000 --- a/Misc/NEWS.d/next/Library/2025-12-17-04-10-35.gh-issue-142831.ee3t4L.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash in the :mod:`json` module where a use-after-free could occur if -the object being encoded is modified during serialization. diff --git a/Misc/NEWS.d/next/Library/2025-12-21-17-56-37.gh-issue-143008.aakErJ.rst b/Misc/NEWS.d/next/Library/2025-12-21-17-56-37.gh-issue-143008.aakErJ.rst new file mode 100644 index 000000000000000..907cdb770ab65f9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-21-17-56-37.gh-issue-143008.aakErJ.rst @@ -0,0 +1,2 @@ +Fix crash in :class:`io.TextIOWrapper` when reentrant +:meth:`io.TextIOBase.detach` is called reentrantly from the underlying buffer. diff --git a/Misc/NEWS.d/next/Library/2026-01-18-06-42-47.gh-issue-143988.MtLtCP.rst b/Misc/NEWS.d/next/Library/2026-01-18-06-42-47.gh-issue-143988.MtLtCP.rst new file mode 100644 index 000000000000000..fcc0cb54934b90e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-18-06-42-47.gh-issue-143988.MtLtCP.rst @@ -0,0 +1,2 @@ +Fixed crashes in :meth:`socket.socket.sendmsg` and :meth:`socket.socket.recvmsg_into` +that could occur if buffer sequences are concurrently mutated. diff --git a/Misc/NEWS.d/next/Library/2026-01-19-21-23-18.gh-issue-105936.dGrzjM.rst b/Misc/NEWS.d/next/Library/2026-01-19-21-23-18.gh-issue-105936.dGrzjM.rst deleted file mode 100644 index c1d3ec806e597cd..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-01-19-21-23-18.gh-issue-105936.dGrzjM.rst +++ /dev/null @@ -1,5 +0,0 @@ -Attempting to mutate non-field attributes of :mod:`dataclasses` -with both *frozen* and *slots* being ``True`` now raises -:class:`~dataclasses.FrozenInstanceError` instead of :class:`TypeError`. -Their non-dataclass subclasses can now freely mutate non-field attributes, -and the original non-slotted class can be garbage collected. diff --git a/Misc/NEWS.d/next/Library/2026-02-07-12-54-20.gh-issue-70647.Bja_Lk.rst b/Misc/NEWS.d/next/Library/2026-02-07-12-54-20.gh-issue-70647.Bja_Lk.rst deleted file mode 100644 index 9fd39743ca58bfc..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-02-07-12-54-20.gh-issue-70647.Bja_Lk.rst +++ /dev/null @@ -1,3 +0,0 @@ -:meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the -format string contains ``%d`` without a year directive. -Using ``%e`` without a year now emits a :exc:`DeprecationWarning`. diff --git a/Misc/NEWS.d/next/Library/2026-02-12-18-05-16.gh-issue-137855.2_PTbg.rst b/Misc/NEWS.d/next/Library/2026-02-12-18-05-16.gh-issue-137855.2_PTbg.rst deleted file mode 100644 index 586c7d3495ae264..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-02-12-18-05-16.gh-issue-137855.2_PTbg.rst +++ /dev/null @@ -1 +0,0 @@ -Reduce the import time of :mod:`dataclasses` module by ~20%. diff --git a/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst b/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst deleted file mode 100644 index 1c2e06c86f65883..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-02-22-00-00-00.gh-issue-145105.csv-reader-reentrant.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator -that calls :func:`next` on the same reader from within ``__next__``. diff --git a/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst b/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst deleted file mode 100644 index 45be0109677cd10..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-02-22-19-36-00.gh-issue-145056.TH8nX4.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for :class:`frozendict` in :meth:`dataclasses.asdict` and :meth:`dataclasses.astuple`. diff --git a/Misc/NEWS.d/next/Library/2026-02-25-10-00-00.gh-issue-145200.m_4PAtcI.rst b/Misc/NEWS.d/next/Library/2026-02-25-10-00-00.gh-issue-145200.m_4PAtcI.rst deleted file mode 100644 index 2fae260377cf73d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-02-25-10-00-00.gh-issue-145200.m_4PAtcI.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`hashlib`: fix a memory leak when allocating -or initializing an OpenSSL HMAC context fails. diff --git a/Misc/NEWS.d/next/Library/2026-02-25-22-20-00.gh-issue-117716.w6kYp9.rst b/Misc/NEWS.d/next/Library/2026-02-25-22-20-00.gh-issue-117716.w6kYp9.rst deleted file mode 100644 index 77182a3f2c33034..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-02-25-22-20-00.gh-issue-117716.w6kYp9.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :mod:`wave` writing of odd-sized ``data`` chunks by appending the required RIFF pad byte and correcting the RIFF chunk size field accordingly. diff --git a/Misc/NEWS.d/next/Library/2026-02-26-12-00-00.gh-issue-145244.Kj31cp.rst b/Misc/NEWS.d/next/Library/2026-02-26-12-00-00.gh-issue-145244.Kj31cp.rst deleted file mode 100644 index 07d7c1fe85e2922..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-02-26-12-00-00.gh-issue-145244.Kj31cp.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a use-after-free in :mod:`json` encoder when a ``default`` callback -mutates the dictionary being serialized. diff --git a/Misc/NEWS.d/next/Library/2026-03-01-01-58-10.gh-issue-145378.oy6rb9.rst b/Misc/NEWS.d/next/Library/2026-03-01-01-58-10.gh-issue-145378.oy6rb9.rst deleted file mode 100644 index b6a6273d882d797..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-01-01-58-10.gh-issue-145378.oy6rb9.rst +++ /dev/null @@ -1 +0,0 @@ -Use ``PyREPL`` as the default input console for :mod:`pdb` diff --git a/Misc/NEWS.d/next/Library/2026-03-11-15-09-52.gh-issue-145831._sW94w.rst b/Misc/NEWS.d/next/Library/2026-03-11-15-09-52.gh-issue-145831._sW94w.rst deleted file mode 100644 index 454b62bc0db95fb..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-11-15-09-52.gh-issue-145831._sW94w.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`!email.quoprimime.decode` leaving a stray ``\r`` when -``eol='\r\n'`` by stripping the full *eol* string instead of one character. diff --git a/Misc/NEWS.d/next/Library/2026-03-12-00-00-00.gh-issue-145846.UbHxjv.rst b/Misc/NEWS.d/next/Library/2026-03-12-00-00-00.gh-issue-145846.UbHxjv.rst deleted file mode 100644 index 63cdb7686da998c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-12-00-00-00.gh-issue-145846.UbHxjv.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix memory leak in ``_lsprof`` when ``clear()`` is called during active -profiling with nested calls. ``clearEntries()`` now walks the entire -``currentProfilerContext`` linked list instead of only freeing the top context. diff --git a/Misc/NEWS.d/next/Library/2026-03-22-16-52-04.gh-issue-146292.rJvvs0.rst b/Misc/NEWS.d/next/Library/2026-03-22-16-52-04.gh-issue-146292.rJvvs0.rst deleted file mode 100644 index 40f3b386155cfc3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-22-16-52-04.gh-issue-146292.rJvvs0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add colour to :mod:`~http.server.BaseHTTPRequestHandler` logs, as used by -the :mod:`http.server` CLI. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-03-22-23-42-22.gh-issue-146313.RtDeAd.rst b/Misc/NEWS.d/next/Library/2026-03-22-23-42-22.gh-issue-146313.RtDeAd.rst deleted file mode 100644 index 1beea3694c422eb..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-22-23-42-22.gh-issue-146313.RtDeAd.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix a deadlock in :mod:`multiprocessing`'s resource tracker -where the parent process could hang indefinitely in :func:`os.waitpid` -during interpreter shutdown if a child created via :func:`os.fork` still -held the resource tracker's pipe open. diff --git a/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst b/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst new file mode 100644 index 000000000000000..99f3cce33497a12 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-09-30-00.gh-issue-146452.Y2N6qZ8J.rst @@ -0,0 +1,2 @@ +Fix segfault in :mod:`pickle` when pickling a dictionary concurrently +mutated by another thread in the free-threaded build. diff --git a/Misc/NEWS.d/next/Library/2026-03-28-11-31-32.gh-issue-146563.cXtSym.rst b/Misc/NEWS.d/next/Library/2026-03-28-11-31-32.gh-issue-146563.cXtSym.rst deleted file mode 100644 index 2103024b616d4e0..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-28-11-31-32.gh-issue-146563.cXtSym.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`xml.parsers.expat`: add an exception note when a custom Expat handler -return value cannot be properly interpreted. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-03-29-21-31-14.gh-issue-146609.BnshCt.rst b/Misc/NEWS.d/next/Library/2026-03-29-21-31-14.gh-issue-146609.BnshCt.rst deleted file mode 100644 index 854fcc32ab76e14..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-03-29-21-31-14.gh-issue-146609.BnshCt.rst +++ /dev/null @@ -1 +0,0 @@ -Add colour to :mod:`timeit` CLI output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-04-01-07-10-49.gh-issue-147957.QXf5Xx.rst b/Misc/NEWS.d/next/Library/2026-04-01-07-10-49.gh-issue-147957.QXf5Xx.rst deleted file mode 100644 index 4a0e15d01c72b6b..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-01-07-10-49.gh-issue-147957.QXf5Xx.rst +++ /dev/null @@ -1 +0,0 @@ -Guarantees that :meth:`collections.UserDict.popitem` will pop in the same order as the wrapped dictionary rather than an arbitrary order. diff --git a/Misc/NEWS.d/next/Library/2026-04-02-05-06-34.gh-issue-147991.2ANtR5.rst b/Misc/NEWS.d/next/Library/2026-04-02-05-06-34.gh-issue-147991.2ANtR5.rst deleted file mode 100644 index 581c52926c3565d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-02-05-06-34.gh-issue-147991.2ANtR5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve :mod:`tomllib` import time (up to 10x faster). Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-04-20-22-02.gh-issue-148100.lSmGQi.rst b/Misc/NEWS.d/next/Library/2026-04-04-20-22-02.gh-issue-148100.lSmGQi.rst deleted file mode 100644 index dd5bc6022e0e5c8..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-04-20-22-02.gh-issue-148100.lSmGQi.rst +++ /dev/null @@ -1,3 +0,0 @@ -:term:`Soft deprecate <soft deprecated>` :func:`re.match` and -:meth:`re.Pattern.match` in favour of :func:`re.prefixmatch` and -:meth:`re.Pattern.prefixmatch`. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst b/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst deleted file mode 100644 index dd88be0ad25d114..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-07-12-37-53.gh-issue-148207.YhGem4.rst +++ /dev/null @@ -1,3 +0,0 @@ -:class:`typing.TypeVarTuple` now accepts ``bound``, ``covariant``, -``contravariant``, and ``infer_variance`` parameters, matching the interface -of :class:`typing.TypeVar` and :class:`typing.ParamSpec`. diff --git a/Misc/NEWS.d/next/Library/2026-04-07-14-13-40.gh-issue-148192.34AUYQ.rst b/Misc/NEWS.d/next/Library/2026-04-07-14-13-40.gh-issue-148192.34AUYQ.rst deleted file mode 100644 index 87a568b50c17c39..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-07-14-13-40.gh-issue-148192.34AUYQ.rst +++ /dev/null @@ -1,3 +0,0 @@ -``email.generator.Generator._make_boundary`` could fail to detect a duplicate -boundary string if linesep was not \n. It now correctly detects boundary -strings when linesep is \r\n as well. diff --git a/Misc/NEWS.d/next/Library/2026-04-08-11-44-12.gh-issue-148225.H34yJp.rst b/Misc/NEWS.d/next/Library/2026-04-08-11-44-12.gh-issue-148225.H34yJp.rst deleted file mode 100644 index 36d79a9fce75afe..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-08-11-44-12.gh-issue-148225.H34yJp.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :mod:`profiling.sampling` ``replay`` command now rejects non-binary -profile files with a clear error explaining that replay only accepts files -created with ``--binary``. diff --git a/Misc/NEWS.d/next/Library/2026-04-08-14-19-17.gh-issue-148241.fO_QT4.rst b/Misc/NEWS.d/next/Library/2026-04-08-14-19-17.gh-issue-148241.fO_QT4.rst deleted file mode 100644 index bf8d0e4382e6f6a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-08-14-19-17.gh-issue-148241.fO_QT4.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`json`: Fix serialization: no longer call ``str(obj)`` on :class:`str` -subclasses. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-09-12-42-42.gh-issue-148254.Xt7vKs.rst b/Misc/NEWS.d/next/Library/2026-04-09-12-42-42.gh-issue-148254.Xt7vKs.rst deleted file mode 100644 index 818310c31b9de6f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-09-12-42-42.gh-issue-148254.Xt7vKs.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use singular "sec" instead of "secs" in :mod:`timeit` verbose output for -consistency with other time units. diff --git a/Misc/NEWS.d/next/Library/2026-04-10-20-23-22.gh-issue-148352.lrec3W.rst b/Misc/NEWS.d/next/Library/2026-04-10-20-23-22.gh-issue-148352.lrec3W.rst deleted file mode 100644 index 508b36791ea0e2f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-10-20-23-22.gh-issue-148352.lrec3W.rst +++ /dev/null @@ -1 +0,0 @@ -Add more color to :mod:`calendar`'s CLI output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-04-11-12-32-38.gh-issue-137855.tsVny_.rst b/Misc/NEWS.d/next/Library/2026-04-11-12-32-38.gh-issue-137855.tsVny_.rst deleted file mode 100644 index 0ec08c2d2fd54ab..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-11-12-32-38.gh-issue-137855.tsVny_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve import time of :mod:`dataclasses` module by lazy importing :mod:`re` -and :mod:`copy` modules. diff --git a/Misc/NEWS.d/next/Library/2026-04-11-17-28-06.gh-issue-127012.h3rLYS.rst b/Misc/NEWS.d/next/Library/2026-04-11-17-28-06.gh-issue-127012.h3rLYS.rst deleted file mode 100644 index eafefb8a6c0cf54..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-11-17-28-06.gh-issue-127012.h3rLYS.rst +++ /dev/null @@ -1,2 +0,0 @@ -``importlib.abc.Traversable.read_text`` now allows/solicits an -``errors`` parameter. diff --git a/Misc/NEWS.d/next/Library/2026-04-12-12-31-45.gh-issue-121190.O6-E5_.rst b/Misc/NEWS.d/next/Library/2026-04-12-12-31-45.gh-issue-121190.O6-E5_.rst deleted file mode 100644 index 1e18015ed9cd437..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-12-12-31-45.gh-issue-121190.O6-E5_.rst +++ /dev/null @@ -1,2 +0,0 @@ -``importlib.resources.files()`` now emits a more meaningful error message -when module spec is None (as found in some ``__main__`` modules). diff --git a/Misc/NEWS.d/next/Library/2026-04-12-16-40-11.gh-issue-148370.0Li2EK.rst b/Misc/NEWS.d/next/Library/2026-04-12-16-40-11.gh-issue-148370.0Li2EK.rst deleted file mode 100644 index 3bb662350796f60..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-12-16-40-11.gh-issue-148370.0Li2EK.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`configparser`: prevent quadratic behavior when a :exc:`~configparser.ParsingError` -is raised after a parser fails to parse multiple lines. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst b/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst deleted file mode 100644 index 85b99531d033b1c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-13-06-22-27.gh-issue-148464.Bj_NZy.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add missing ``__ctype_le/be__`` attributes for -:class:`~ctypes.c_float_complex` and :class:`~ctypes.c_double_complex`. Patch -by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2026-04-13-15-59-44.gh-issue-148518.RQdvsu.rst b/Misc/NEWS.d/next/Library/2026-04-13-15-59-44.gh-issue-148518.RQdvsu.rst deleted file mode 100644 index 994e4ad74466700..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-13-15-59-44.gh-issue-148518.RQdvsu.rst +++ /dev/null @@ -1,4 +0,0 @@ -If an email containing an address header that ended in an open double quote -was parsed with a non-``compat32`` policy, accessing the ``username`` attribute -of the mailbox accessed through that header object would result in an -``IndexError``. It now correctly returns an empty string as the result. diff --git a/Misc/NEWS.d/next/Library/2026-04-13-21-38-50.gh-issue-144881.3kPqXw.rst b/Misc/NEWS.d/next/Library/2026-04-13-21-38-50.gh-issue-144881.3kPqXw.rst deleted file mode 100644 index 0812dc9efb6d8b1..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-13-21-38-50.gh-issue-144881.3kPqXw.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`asyncio` debugging tools (``python -m asyncio ps`` and ``pstree``) -now retry automatically on transient errors that can occur when attaching -to a process under active thread delegation. The number of retries can be -controlled with the ``--retries`` flag. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2026-04-14-09-04-35.gh-issue-148508.-GiXml.rst b/Misc/NEWS.d/next/Library/2026-04-14-09-04-35.gh-issue-148508.-GiXml.rst deleted file mode 100644 index 7995dec397f7b7e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-14-09-04-35.gh-issue-148508.-GiXml.rst +++ /dev/null @@ -1,2 +0,0 @@ -An intermittent timing error when running SSL tests on iOS has been -resolved. diff --git a/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst b/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst deleted file mode 100644 index 44216318d474a9f..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-15-11-00-39.gh-issue-146553.VGOsoP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix infinite loop in :func:`typing.get_type_hints` when ``__wrapped__`` -forms a cycle. Patch by Shamil Abdulaev. diff --git a/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst b/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst deleted file mode 100644 index d7acdb0983837a9..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-15-20-32-55.gh-issue-148639.-dwsjB.rst +++ /dev/null @@ -1,2 +0,0 @@ -Implement :pep:`800`, adding the :deco:`typing.disjoint_base` decorator. -Patch by Jelle Zijlstra. diff --git a/Misc/NEWS.d/next/Library/2026-04-15-21-46-52.gh-issue-148641.-aoFyC.rst b/Misc/NEWS.d/next/Library/2026-04-15-21-46-52.gh-issue-148641.-aoFyC.rst deleted file mode 100644 index 53779676fd100c5..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-15-21-46-52.gh-issue-148641.-aoFyC.rst +++ /dev/null @@ -1,3 +0,0 @@ -:pep:`829` (package startup configuration files) implements a new format -``<name>.start`` parallel to ``<name>.pth`` files, to replace ``import`` -lines in the latter. diff --git a/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst b/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst deleted file mode 100644 index b69f94a17663dc6..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-16-13-30-00.gh-issue-148651.ZsTdLk.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix reference leak in :class:`compression.zstd.ZstdDecompressor` when an -invalid option key is passed. diff --git a/Misc/NEWS.d/next/Library/2026-04-17-13-56-44.gh-issue-148675.f1kG70.rst b/Misc/NEWS.d/next/Library/2026-04-17-13-56-44.gh-issue-148675.f1kG70.rst deleted file mode 100644 index 340168819e891bf..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-17-13-56-44.gh-issue-148675.f1kG70.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`array`, :mod:`struct`: Add support for ``Zd`` and ``Zf`` formats for -double complex and float complex. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst b/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst deleted file mode 100644 index 1e367716e5a0a70..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-17-16-31-58.gh-issue-148688.vVugFn.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`bz2`, :mod:`compression.zstd`, :mod:`lzma`, :mod:`zlib`: Fix a double -free on memory allocation failure. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst b/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst deleted file mode 100644 index 7e49cedda7beb2e..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-18-17-37-13.gh-issue-148740.sYnFi0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix usage for :mod:`uuid` command-line interface to support a custom namespace be -provided for uuid3 and uuid5. diff --git a/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst b/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst deleted file mode 100644 index db5e94c0ccac501..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-18-21-39-15.gh-issue-148735.siw6DG.rst +++ /dev/null @@ -1,3 +0,0 @@ -:mod:`xml.etree.ElementTree`: Fix a use-after-free in -:meth:`Element.findtext <xml.etree.ElementTree.Element.findtext>` when the -element tree is mutated concurrently during the search. diff --git a/Misc/NEWS.d/next/Library/2026-04-20-01-24-22.gh-issue-92455.vXhmad.rst b/Misc/NEWS.d/next/Library/2026-04-20-01-24-22.gh-issue-92455.vXhmad.rst new file mode 100644 index 000000000000000..8d2a11cb7761377 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-20-01-24-22.gh-issue-92455.vXhmad.rst @@ -0,0 +1,3 @@ +Fix :mod:`mimetypes` to prefer case-sensitive matches for suffix mappings and +MIME type suffixes before falling back to case-insensitive matches. +Contributed by Xiao Yuan. diff --git a/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst b/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst deleted file mode 100644 index 6fcd30e8f057b95..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-20-18-29-21.gh-issue-148801.ROeNqs.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ -<object.__deepcopy__>` on deeply nested trees. diff --git a/Misc/NEWS.d/next/Library/2026-04-21-06-30-59.gh-issue-119670.pMWZfY.rst b/Misc/NEWS.d/next/Library/2026-04-21-06-30-59.gh-issue-119670.pMWZfY.rst new file mode 100644 index 000000000000000..fc1941be4dc7925 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-21-06-30-59.gh-issue-119670.pMWZfY.rst @@ -0,0 +1,2 @@ +Add keyword-only parameter *force* to :func:`shlex.quote` to force quoting +a string, even if it is already safe for a shell without being quoted. diff --git a/Misc/NEWS.d/next/Library/2026-04-22-20-49-49.gh-issue-124397.plMglV.rst b/Misc/NEWS.d/next/Library/2026-04-22-20-49-49.gh-issue-124397.plMglV.rst deleted file mode 100644 index 431448a484b45f7..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-22-20-49-49.gh-issue-124397.plMglV.rst +++ /dev/null @@ -1,3 +0,0 @@ -The threading module added tooling to support concurrent iterator access: -:class:`threading.serialize_iterator`, :func:`threading.synchronized_iterator`, -and :func:`threading.concurrent_tee`. diff --git a/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst b/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst deleted file mode 100644 index d3790079545a072..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-23-07-38-04.gh-issue-148680.___ePl.rst +++ /dev/null @@ -1 +0,0 @@ -``ForwardRef`` objects that contain internal names to represent known objects now show the ``type_repr`` of the known object rather than the internal ``__annotationlib_name_x__`` name when evaluated as strings. diff --git a/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst b/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst new file mode 100644 index 000000000000000..762815270e4d403 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-23-12-50-15.gh-issue-148441.zvpCkR.rst @@ -0,0 +1,4 @@ +:mod:`xml.parsers.expat`: prevent a crash in +:meth:`~xml.parsers.expat.xmlparser.CharacterDataHandler` +when the character data size exceeds the parser's +:attr:`buffer size <xml.parsers.expat.xmlparser.buffer_size>`. diff --git a/Misc/NEWS.d/next/Library/2026-04-23-21-47-49.gh-issue-148947.W4V2lG.rst b/Misc/NEWS.d/next/Library/2026-04-23-21-47-49.gh-issue-148947.W4V2lG.rst deleted file mode 100644 index f9783266f5cc42c..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-23-21-47-49.gh-issue-148947.W4V2lG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix crash in :deco:`dataclasses.dataclass` with ``slots=True`` that occurred -when a function found within the class had an empty ``__class__`` cell. diff --git a/Misc/NEWS.d/next/Library/2026-04-24-19-54-00.gh-issue-148954.v1.rst b/Misc/NEWS.d/next/Library/2026-04-24-19-54-00.gh-issue-148954.v1.rst new file mode 100644 index 000000000000000..6245af7e362e920 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-24-19-54-00.gh-issue-148954.v1.rst @@ -0,0 +1 @@ +Fix XML injection vulnerability in :func:`xmlrpc.client.dumps` where the ``methodname`` was not being escaped before interpolation into the XML body. diff --git a/Misc/NEWS.d/next/Library/2026-04-25-11-56-05.gh-issue-146311.iHWO0v.rst b/Misc/NEWS.d/next/Library/2026-04-25-11-56-05.gh-issue-146311.iHWO0v.rst deleted file mode 100644 index 4f4a8365b6cf5e5..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-25-11-56-05.gh-issue-146311.iHWO0v.rst +++ /dev/null @@ -1,7 +0,0 @@ -Add a *canonical* keyword-only parameter to the base16, base32, base64, -base85, ascii85, and Z85 decoders in :mod:`base64` and :mod:`binascii`. -When true, encodings with non-zero padding bits (base16/32/64) or -non-canonical encodings (base85/ascii85) are rejected. Single-character -final groups in :func:`binascii.a2b_ascii85` and :func:`binascii.a2b_base85` -are now always rejected as encoding violations, regardless of *canonical*; -previously they were silently ignored and produced no output bytes. diff --git a/Misc/NEWS.d/next/Library/2026-04-25-12-04-27.gh-issue-148849.Vk6yEW.rst b/Misc/NEWS.d/next/Library/2026-04-25-12-04-27.gh-issue-148849.Vk6yEW.rst deleted file mode 100644 index 9725d63747d451a..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-25-12-04-27.gh-issue-148849.Vk6yEW.rst +++ /dev/null @@ -1,4 +0,0 @@ -Deprecate :meth:`http.cookies.Morsel.js_output` and -:meth:`http.cookies.BaseCookie.js_output`, which will be removed in -Python 3.19. Use :meth:`http.cookies.Morsel.output` or -:meth:`http.cookies.BaseCookie.output` instead. diff --git a/Misc/NEWS.d/next/Library/2026-04-25-12-50-46.gh-issue-148981.YMM4Y9.rst b/Misc/NEWS.d/next/Library/2026-04-25-12-50-46.gh-issue-148981.YMM4Y9.rst deleted file mode 100644 index e36c7745f4080a5..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-25-12-50-46.gh-issue-148981.YMM4Y9.rst +++ /dev/null @@ -1 +0,0 @@ -Add *color* parameter to :func:`ast.dump`. diff --git a/Misc/NEWS.d/next/Library/2026-04-25-18-09-16.gh-issue-148991.AZ64Et.rst b/Misc/NEWS.d/next/Library/2026-04-25-18-09-16.gh-issue-148991.AZ64Et.rst deleted file mode 100644 index 336ed42e51f1b80..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-25-18-09-16.gh-issue-148991.AZ64Et.rst +++ /dev/null @@ -1 +0,0 @@ -Add colour to :mod:`tokenize` CLI output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-04-26-11-25-38.gh-issue-118158.xIy05H.rst b/Misc/NEWS.d/next/Library/2026-04-26-11-25-38.gh-issue-118158.xIy05H.rst new file mode 100644 index 000000000000000..bd22871c1c0c172 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-26-11-25-38.gh-issue-118158.xIy05H.rst @@ -0,0 +1 @@ +Ensure py_compile CLI error messages end with a newline. Contributed by Xiao Yuan. diff --git a/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst b/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst deleted file mode 100644 index d12a92e9f530da1..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-26-23-01-50.gh-issue-149026.Akk4Bc.rst +++ /dev/null @@ -1 +0,0 @@ -Add colour to :mod:`pickletools` CLI output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-04-27-11-12-00.gh-issue-149046.74shDd.rst b/Misc/NEWS.d/next/Library/2026-04-27-11-12-00.gh-issue-149046.74shDd.rst new file mode 100644 index 000000000000000..b05c4222e30fcd2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-27-11-12-00.gh-issue-149046.74shDd.rst @@ -0,0 +1,2 @@ +:mod:`io`: Fix :class:`io.StringIO` serialization: no longer call ``str(obj)`` on :class:`str` +subclasses. Patch by Thomas Kowalski. diff --git a/Misc/NEWS.d/next/Library/2026-04-27-17-12-11.gh-issue-148914.i5C3kW.rst b/Misc/NEWS.d/next/Library/2026-04-27-17-12-11.gh-issue-148914.i5C3kW.rst deleted file mode 100644 index 8348aad0d892c37..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-27-17-12-11.gh-issue-148914.i5C3kW.rst +++ /dev/null @@ -1,6 +0,0 @@ -Fix memoization of in-band :class:`~pickle.PickleBuffer` in the Python -implementation of :mod:`pickle`. Previously, identical -:class:`!PickleBuffer`\ s did not preserve identity, and empty writable -:class:`!PickleBuffer` memoized an empty bytearray object in place of -``b''``, so the following references to ``b''`` were unpickled as an empty -bytearray object. diff --git a/Misc/NEWS.d/next/Library/2026-04-27-20-15-54.gh-issue-149083.BdrpU8.rst b/Misc/NEWS.d/next/Library/2026-04-27-20-15-54.gh-issue-149083.BdrpU8.rst deleted file mode 100644 index 7ad816168021169..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-27-20-15-54.gh-issue-149083.BdrpU8.rst +++ /dev/null @@ -1,2 +0,0 @@ -:data:`dataclasses.MISSING` and :data:`dataclasses.KW_ONLY` are now -instances of :class:`sentinel`. diff --git a/Misc/NEWS.d/next/Library/2026-04-27-22-34-09.gh-issue-148093.9pWceM.rst b/Misc/NEWS.d/next/Library/2026-04-27-22-34-09.gh-issue-148093.9pWceM.rst deleted file mode 100644 index 9418044201f8bd2..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-27-22-34-09.gh-issue-148093.9pWceM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix an out-of-bounds read of one byte in :func:`binascii.a2b_uu`. Raise -:exc:`binascii.Error`, instead of reading past the buffer end. diff --git a/Misc/NEWS.d/next/Library/2026-04-28-16-25-40.gh-issue-148641.aFgym0.rst b/Misc/NEWS.d/next/Library/2026-04-28-16-25-40.gh-issue-148641.aFgym0.rst deleted file mode 100644 index 3820c5f9d0e697d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-28-16-25-40.gh-issue-148641.aFgym0.rst +++ /dev/null @@ -1,2 +0,0 @@ -:func:`pkgutil.resolve_name` gets a new optional, keyword-only argument -called ``strict``. The default is ``False`` for backward compatibility. diff --git a/Misc/NEWS.d/next/Library/2026-04-28-16-30-48.gh-issue-149085.5aNgBD.rst b/Misc/NEWS.d/next/Library/2026-04-28-16-30-48.gh-issue-149085.5aNgBD.rst deleted file mode 100644 index a5b92287bd0ef85..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-28-16-30-48.gh-issue-149085.5aNgBD.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add a *max_threads* keyword argument to :func:`faulthandler.dump_traceback`, -:func:`faulthandler.dump_traceback_later`, :func:`faulthandler.enable`, and -:func:`faulthandler.register`. diff --git a/Misc/NEWS.d/next/Library/2026-04-28-17-47-55.gh-issue-148292.oIq3ml.rst b/Misc/NEWS.d/next/Library/2026-04-28-17-47-55.gh-issue-148292.oIq3ml.rst deleted file mode 100644 index e1f308df5a678e6..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-28-17-47-55.gh-issue-148292.oIq3ml.rst +++ /dev/null @@ -1,7 +0,0 @@ -:mod:`ssl`: Update :class:`ssl.SSLSocket` and :class:`ssl.SSLObject` for -OpenSSL 4. The classes now remember if they get a :exc:`ssl.SSLEOFError`. In this -case, following :meth:`~ssl.SSLSocket.read`, :meth:`!sendfile`, -:meth:`~ssl.SSLSocket.write`, and :meth:`~ssl.SSLSocket.do_handshake` calls -raise :exc:`ssl.SSLEOFError` without calling the underlying OpenSSL function. -Thanks to that, :class:`ssl.SSLSocket` behaves the same on all OpenSSL versions -on EOF. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-29-08-10-17.gh-issue-149056.jnaD4W.rst b/Misc/NEWS.d/next/Library/2026-04-29-08-10-17.gh-issue-149056.jnaD4W.rst new file mode 100644 index 000000000000000..0026d02c8762570 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-29-08-10-17.gh-issue-149056.jnaD4W.rst @@ -0,0 +1,2 @@ +Fix :func:`json.load` not forwarding the *array_hook* argument to +:func:`json.loads`. Patch by Thomas Kowalski. diff --git a/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst b/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst deleted file mode 100644 index 06186773474fb92..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-29-14-33-42.gh-issue-149148.EaiYvk.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`ensurepip`: Upgrade bundled pip to 26.1. This version fixes -the :cve:`2026-3219` vulnerability. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-04-29-16-11-27.gh-issue-149117.yEeTYd.rst b/Misc/NEWS.d/next/Library/2026-04-29-16-11-27.gh-issue-149117.yEeTYd.rst deleted file mode 100644 index 41223e90ed0b0e3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-29-16-11-27.gh-issue-149117.yEeTYd.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :func:`runpy.run_module` and :func:`runpy.run_path` to set the -:attr:`~ImportError.name` attribute on the :exc:`ImportError` they -raise. diff --git a/Misc/NEWS.d/next/Library/2026-04-30-14-21-26.gh-issue-149173.KJqZm0.rst b/Misc/NEWS.d/next/Library/2026-04-30-14-21-26.gh-issue-149173.KJqZm0.rst deleted file mode 100644 index 019ab76b8635776..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-04-30-14-21-26.gh-issue-149173.KJqZm0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix inverted :envvar:`PYTHON_BASIC_REPL` environment check in -``pdb._pyrepl_available``. diff --git a/Misc/NEWS.d/next/Library/2026-05-01-10-20-27.gh-issue-149214.btP546.rst b/Misc/NEWS.d/next/Library/2026-05-01-10-20-27.gh-issue-149214.btP546.rst deleted file mode 100644 index cbb05620626d0f0..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-01-10-20-27.gh-issue-149214.btP546.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix :mod:`!_remote_debugging` misreading non-ASCII Unicode strings (Latin-1, -BMP and non-BMP) from a remote process. Filenames and function names that -contain non-ASCII characters are now reported correctly in stack traces, the -sampling profiler, and :mod:`asyncio` task introspection. diff --git a/Misc/NEWS.d/next/Library/2026-05-01-11-39-37.gh-issue-143231.0cOHET.rst b/Misc/NEWS.d/next/Library/2026-05-01-11-39-37.gh-issue-143231.0cOHET.rst deleted file mode 100644 index 05c9fa799041545..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-01-11-39-37.gh-issue-143231.0cOHET.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`unittest.TestCase.assertWarns` and -:func:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that -do not match the specified category or regex. -Nested context managers are now supported. diff --git a/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst b/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst deleted file mode 100644 index c265b54db8bed47..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-01-16-45-31.gh-issue-149231.x2nBEE.rst +++ /dev/null @@ -1 +0,0 @@ -In :mod:`tomllib`, the number of parts in TOML keys is now limited. diff --git a/Misc/NEWS.d/next/Library/2026-05-02-01-09-29.gh-issue-149221.__KOks.rst b/Misc/NEWS.d/next/Library/2026-05-02-01-09-29.gh-issue-149221.__KOks.rst deleted file mode 100644 index fab2b0f6a234891..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-02-01-09-29.gh-issue-149221.__KOks.rst +++ /dev/null @@ -1 +0,0 @@ -Catch rare math domain error for :func:`random.binomialvariate`. diff --git a/Misc/NEWS.d/next/Library/2026-05-02-12-30-35.gh-issue-148675.cu2YFT.rst b/Misc/NEWS.d/next/Library/2026-05-02-12-30-35.gh-issue-148675.cu2YFT.rst deleted file mode 100644 index 0f4a737b3ce5606..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-02-12-30-35.gh-issue-148675.cu2YFT.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :data:`array.typecodes` type changed from :class:`str` to :class:`tuple` -to support type codes longer than 1 character (``Zf`` and ``Zd``). Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-05-02-15-58-08.gh-issue-148675.b3ZNlj.rst b/Misc/NEWS.d/next/Library/2026-05-02-15-58-08.gh-issue-148675.b3ZNlj.rst deleted file mode 100644 index 93fd2c44cd64483..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-02-15-58-08.gh-issue-148675.b3ZNlj.rst +++ /dev/null @@ -1,4 +0,0 @@ -:mod:`ctypes`: Change the :py:attr:`~ctypes._SimpleCData._type_` of -:class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and -:class:`~ctypes.c_longdouble_complex` from ``F``, ``D`` and ``G`` to ``Zf``, -``Zd`` and ``Zg`` for compatibility with numpy. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-05-02-18-23-50.gh-issue-143231.oBbQb5.rst b/Misc/NEWS.d/next/Library/2026-05-02-18-23-50.gh-issue-143231.oBbQb5.rst deleted file mode 100644 index e4769866c2045dc..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-02-18-23-50.gh-issue-143231.oBbQb5.rst +++ /dev/null @@ -1 +0,0 @@ -A *module* attribute has been added to :class:`!warnings.WarningMessage`. diff --git a/Misc/NEWS.d/next/Library/2026-05-02-19-09-04.gh-issue-149296.DuKF0j.rst b/Misc/NEWS.d/next/Library/2026-05-02-19-09-04.gh-issue-149296.DuKF0j.rst deleted file mode 100644 index f238693720696ba..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-02-19-09-04.gh-issue-149296.DuKF0j.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add a ``dump`` subcommand to :mod:`profiling.sampling` that prints a single -traceback-style snapshot of a running process's Python stack, including -per-thread status, source line highlighting, optional bytecode opcode names, -and async-aware task reconstruction. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Library/2026-05-03-23-29-34.gh-issue-142389.SVYiSv.rst b/Misc/NEWS.d/next/Library/2026-05-03-23-29-34.gh-issue-142389.SVYiSv.rst deleted file mode 100644 index 6f3f53187dead7d..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-03-23-29-34.gh-issue-142389.SVYiSv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add backticks for colour to regrtest and pdb's help description. Patch by -Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-05-03-23-47-59.gh-issue-146609.V9jqYf.rst b/Misc/NEWS.d/next/Library/2026-05-03-23-47-59.gh-issue-146609.V9jqYf.rst deleted file mode 100644 index 51fde3b42494ecd..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-03-23-47-59.gh-issue-146609.V9jqYf.rst +++ /dev/null @@ -1 +0,0 @@ -Use :mod:`argparse` for colour help :mod:`timeit` CLI. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-05-04-00-51-32.gh-issue-149010.BCp_8k.rst b/Misc/NEWS.d/next/Library/2026-05-04-00-51-32.gh-issue-149010.BCp_8k.rst deleted file mode 100644 index 31bf235566744c3..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-05-04-00-51-32.gh-issue-149010.BCp_8k.rst +++ /dev/null @@ -1,6 +0,0 @@ -The ``inspect`` module CLI now reports as much information as it has -available for non-source modules when ``--details`` is specified, and -provides an error message rather than a traceback when ``--details`` is -omitted. It also reports improved information when the given target location -is not the target's defining location and when the given target is a data -value rather than a class or function definition. diff --git a/Misc/NEWS.d/next/Library/2026-05-06-20-10-44.gh-issue-149464.3mPhcr.rst b/Misc/NEWS.d/next/Library/2026-05-06-20-10-44.gh-issue-149464.3mPhcr.rst new file mode 100644 index 000000000000000..9f987d2a662d8fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-06-20-10-44.gh-issue-149464.3mPhcr.rst @@ -0,0 +1,3 @@ +Add :func:`os.pidfd_getfd` for duplicating a file descriptor from another +process via a pidfd. Available on Linux 5.6+. Patch by Maurycy +Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-05-07-14-18-47.gh-issue-149489.bX9iHe.rst b/Misc/NEWS.d/next/Library/2026-05-07-14-18-47.gh-issue-149489.bX9iHe.rst new file mode 100644 index 000000000000000..1550c893fd7c45b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-07-14-18-47.gh-issue-149489.bX9iHe.rst @@ -0,0 +1,5 @@ +Fix :mod:`~xml.etree.ElementTree` serialization to HTML. The content of +comments, processing instructions and elements "xmp", "iframe", "noembed", +"noframes", and "plaintext" is no longer escaped. The "plaintext" element no +longer have the closing tag. Add support of empty attributes (with value +``None``). diff --git a/Misc/NEWS.d/next/Library/2026-05-07-17-35-47.gh-issue-149499.G2ER-R.rst b/Misc/NEWS.d/next/Library/2026-05-07-17-35-47.gh-issue-149499.G2ER-R.rst new file mode 100644 index 000000000000000..796d39f5ba05bf1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-07-17-35-47.gh-issue-149499.G2ER-R.rst @@ -0,0 +1,3 @@ +Removed the :func:`!sysconfig.expand_makefile_vars` function which has been +deprecated since Python 3.14. Use the ``vars`` argument of +:func:`sysconfig.get_paths` instead. diff --git a/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst b/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst new file mode 100644 index 000000000000000..4a1c6f3f5b4e579 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-07-21-58-17.gh-issue-149388.DDBPeA.rst @@ -0,0 +1 @@ +Make :class:`!asyncio.windows_utils.PipeHandle` closing idempotent. diff --git a/Misc/NEWS.d/next/Library/2026-05-08-06-12-26.gh-issue-149528.AxdDxa.rst b/Misc/NEWS.d/next/Library/2026-05-08-06-12-26.gh-issue-149528.AxdDxa.rst new file mode 100644 index 000000000000000..b2c6e427a1c592f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-06-12-26.gh-issue-149528.AxdDxa.rst @@ -0,0 +1 @@ +Remove ``annotationlib.ForwardRef._evaludate`` deprecated method. diff --git a/Misc/NEWS.d/next/Library/2026-05-08-06-56-57.gh-issue-149530.cl2AJ8.rst b/Misc/NEWS.d/next/Library/2026-05-08-06-56-57.gh-issue-149530.cl2AJ8.rst new file mode 100644 index 000000000000000..2b74e8cdadd03c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-06-56-57.gh-issue-149530.cl2AJ8.rst @@ -0,0 +1,2 @@ +Removed :meth:`!symtable.Class.get_methods` which has been deprecated since +3.14. diff --git a/Misc/NEWS.d/next/Library/2026-05-08-07-51-54.gh-issue-80480.nnA4p1.rst b/Misc/NEWS.d/next/Library/2026-05-08-07-51-54.gh-issue-80480.nnA4p1.rst new file mode 100644 index 000000000000000..2f7f15b169b26b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-07-51-54.gh-issue-80480.nnA4p1.rst @@ -0,0 +1,2 @@ +Remove deprecated ``'u'`` type code (:c:type:`wchar_t`) for the :mod:`array` +module. Use ``'w'`` format code instead (:c:type:`Py_UCS4`, always 4 bytes). diff --git a/Misc/NEWS.d/next/Library/2026-05-08-08-24-01.gh-issue-149537.hVFVnt.rst b/Misc/NEWS.d/next/Library/2026-05-08-08-24-01.gh-issue-149537.hVFVnt.rst new file mode 100644 index 000000000000000..2fb0359a2ea8936 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-08-24-01.gh-issue-149537.hVFVnt.rst @@ -0,0 +1,2 @@ +Remove kw parameters from python version of :func:`functools.reduce` +function. diff --git a/Misc/NEWS.d/next/Library/2026-05-08-09-11-48.gh-issue-149534.Tw7eeY.rst b/Misc/NEWS.d/next/Library/2026-05-08-09-11-48.gh-issue-149534.Tw7eeY.rst new file mode 100644 index 000000000000000..0938935a75d8c16 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-09-11-48.gh-issue-149534.Tw7eeY.rst @@ -0,0 +1 @@ +Fix merging of :class:`collections.defaultdict` and :class:`frozendict`. diff --git a/Misc/NEWS.d/next/Library/2026-05-08-14-24-16.gh-issue-149436.dEV02X.rst b/Misc/NEWS.d/next/Library/2026-05-08-14-24-16.gh-issue-149436.dEV02X.rst new file mode 100644 index 000000000000000..401cf044e1514fb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-14-24-16.gh-issue-149436.dEV02X.rst @@ -0,0 +1 @@ +Improve performance of :meth:`inspect.getattr_static`. diff --git a/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst b/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst new file mode 100644 index 000000000000000..cfbcde81493e221 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-15-08-35.gh-issue-112821.t9T1YD.rst @@ -0,0 +1,4 @@ +In the REPL, autocompletion might run arbitrary code in the getter of a +descriptor. If that getter raised an exception, autocompletion would fail to +present any options for the entire object. Autocompletion now works as +expected for these objects. diff --git a/Misc/NEWS.d/next/Library/2026-05-08-16-44-20.gh-issue-149567.iiZKEj.rst b/Misc/NEWS.d/next/Library/2026-05-08-16-44-20.gh-issue-149567.iiZKEj.rst new file mode 100644 index 000000000000000..e30563df79b3bcf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-08-16-44-20.gh-issue-149567.iiZKEj.rst @@ -0,0 +1,2 @@ +Remove the :exc:`!shutil.ExecError` exception which has been deprecated +since Python 3.14. diff --git a/Misc/NEWS.d/next/Library/2026-05-09-11-22-29.gh-issue-149595.1nuYXw.rst b/Misc/NEWS.d/next/Library/2026-05-09-11-22-29.gh-issue-149595.1nuYXw.rst new file mode 100644 index 000000000000000..f3971554de37487 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-09-11-22-29.gh-issue-149595.1nuYXw.rst @@ -0,0 +1,2 @@ +Remove the :func:`!sys._enablelegacywindowsfsencoding` function which has +been deprecated since Python 3.13. diff --git a/Misc/NEWS.d/next/Library/2026-05-09-15-17-59.gh-issue-149598.aLrXRw.rst b/Misc/NEWS.d/next/Library/2026-05-09-15-17-59.gh-issue-149598.aLrXRw.rst new file mode 100644 index 000000000000000..8c06ba5e7d5e4e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-09-15-17-59.gh-issue-149598.aLrXRw.rst @@ -0,0 +1 @@ +Remove support of deprecated *strm* argument for :mod:`logging` handlers. diff --git a/Misc/NEWS.d/next/Library/2026-05-09-15-42-54.gh-issue-149600.ZbL-iv.rst b/Misc/NEWS.d/next/Library/2026-05-09-15-42-54.gh-issue-149600.ZbL-iv.rst new file mode 100644 index 000000000000000..732787dc92c1090 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-09-15-42-54.gh-issue-149600.ZbL-iv.rst @@ -0,0 +1 @@ +Remove deprecated :func:`!asyncio.iscoroutinefunction` function. diff --git a/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst b/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst new file mode 100644 index 000000000000000..5169c6c203fc1b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-09-21-02-08.gh-issue-149614.U4snj3.rst @@ -0,0 +1 @@ +Fix a regression that broke the ability to deepcopy :class:`argparse.ArgumentParser` instances. diff --git a/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst b/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst new file mode 100644 index 000000000000000..40fe7e9fd6a0086 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-07-21-51.gh-issue-139489.rS7LTA.rst @@ -0,0 +1 @@ +Add :func:`xml.is_valid_text` to ``xml.__all__``. diff --git a/Misc/NEWS.d/next/Library/2026-05-10-14-10-00.gh-issue-149634.iT5cwC.rst b/Misc/NEWS.d/next/Library/2026-05-10-14-10-00.gh-issue-149634.iT5cwC.rst new file mode 100644 index 000000000000000..73e8d73c36891ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-14-10-00.gh-issue-149634.iT5cwC.rst @@ -0,0 +1 @@ +Remove deprecated and unused :attr:`!tarfile.TarInfo.tarfile` attribute. diff --git a/Misc/NEWS.d/next/Library/2026-05-10-16-06-27.gh-issue-149609.yLKPXo.rst b/Misc/NEWS.d/next/Library/2026-05-10-16-06-27.gh-issue-149609.yLKPXo.rst new file mode 100644 index 000000000000000..3acba7d25021b28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-16-06-27.gh-issue-149609.yLKPXo.rst @@ -0,0 +1,3 @@ +Raise :exc:`DeprecationWarning` on using :class:`abc.abstractclassmethod`, +:class:`abc.abstractstaticmethod`, and :class:`abc.abstractproperty`, +schedule its removal for Python 3.21. diff --git a/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst b/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst new file mode 100644 index 000000000000000..6734250fdd6af3c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-19-26-50.gh-issue-149584.x7Qm9A.rst @@ -0,0 +1,4 @@ +Fix excessive overhead in the Tachyon profiler when inspecting a remote +process by avoiding repeated remote page-cache scans, batching predicted +remote reads, and reusing cached profiler result objects. Patch by Pablo +Galindo and Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst b/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst new file mode 100644 index 000000000000000..88bf268123bbecc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-10-23-51-23.gh-issue-149504.pDSCbn.rst @@ -0,0 +1,5 @@ +Fix :func:`site.addsitedir` to allow re-entrant calls from within startup +files. Previously, a ``.pth`` file containing an ``import`` line that +called :func:`site.addsitedir` (or a ``.start`` entry point doing the same) +could crash with ``RuntimeError: dictionary changed size during iteration`` +during site initialization, breaking tools such as ``uv run --with``. diff --git a/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst b/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst new file mode 100644 index 000000000000000..676d788cbce62a3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-12-06-24-54.gh-issue-149701.8v9RTm.rst @@ -0,0 +1 @@ +Fix bad return code from Lib/venv/bin/activate if hashing is disabled diff --git a/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst b/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst new file mode 100644 index 000000000000000..25344e5a90f022c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-12-13-03-45.gh-issue-149718.SaM1NJ.rst @@ -0,0 +1,4 @@ +Coalesce consecutive identical stack frames in Tachyon, so aggregating +collectors (pstats, collapsed, flamegraph, gecko) receive one collect. +Improves sample rate 3x, error rate and missed rate drop by 70%. Patch by +Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-05-12-14-44-39.gh-issue-149720.ccsoW-.rst b/Misc/NEWS.d/next/Library/2026-05-12-14-44-39.gh-issue-149720.ccsoW-.rst new file mode 100644 index 000000000000000..f16d8770d740f49 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-12-14-44-39.gh-issue-149720.ccsoW-.rst @@ -0,0 +1 @@ +Remove support for undotted *ext* in :meth:`mimetypes.MimeTypes.add_type`. diff --git a/Misc/NEWS.d/next/Library/2026-05-13-12-16-54.gh-issue-149473.nOQZqn.rst b/Misc/NEWS.d/next/Library/2026-05-13-12-16-54.gh-issue-149473.nOQZqn.rst new file mode 100644 index 000000000000000..db624aba31a9de0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-13-12-16-54.gh-issue-149473.nOQZqn.rst @@ -0,0 +1,2 @@ +Calling ``os.environ.clear()`` now emits ``os._clearenv`` auditing event. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst b/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst new file mode 100644 index 000000000000000..f9e8538527d204e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-13-23-18-39.gh-issue-149801.S_FfGr.rst @@ -0,0 +1,2 @@ +Add IANA registered names and aliases with leading zeros before number (like +IBM00858, CP00858, IBM01140, CP01140) for corresponding codecs. diff --git a/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst b/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst new file mode 100644 index 000000000000000..3ea70071ec3c75d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-14-15-55-28.gh-issue-149816.ZaXQ0q.rst @@ -0,0 +1,2 @@ +Fix a race condition in ``_random.Random.__init__`` method in free-threading +mode. diff --git a/Misc/NEWS.d/next/Library/2026-05-14-17-01-19.gh-issue-62259.ytlFD5.rst b/Misc/NEWS.d/next/Library/2026-05-14-17-01-19.gh-issue-62259.ytlFD5.rst new file mode 100644 index 000000000000000..d0af77366378b88 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-14-17-01-19.gh-issue-62259.ytlFD5.rst @@ -0,0 +1,9 @@ +Add support for multiple multi-byte encodings in the :mod:`XML parser +<xml.parsers.expat>`: "cp932", "cp949", "cp950", "Big5","EUC-JP", "GB2312", +"GBK", "johab", and "Shift_JIS". Add partial support (only BMP characters) +for multi-byte encodings "Big5-HKSCS", "EUC_JIS-2004", "EUC_JISX0213", +"Shift_JIS-2004", "Shift_JISX0213", "utf-8-sig" and non-standard aliases +like "UTF8" (without hyphen). The parser now raises :exc:`ValueError` for +known unsupported multi-byte encodings such us "ISO-2022-JP" or +"raw-unicode-escape" instead of failing later, when encounter non-ASCII +data. diff --git a/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst b/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst new file mode 100644 index 000000000000000..66e6da0ecf0d87c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-15-16-28-00.gh-issue-149819.fixpth.rst @@ -0,0 +1,4 @@ +Fix regression in :func:`site.addsitedir` where ``.pth`` files were no +longer processed in Python subprocesses. This happened because +:func:`site.main` seeded ``known_paths`` with entries inherited from +the parent process, causing ``addsitedir`` to skip ``.pth`` processing. diff --git a/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst b/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst new file mode 100644 index 000000000000000..fa667c4110941e9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-15-18-44-20.gh-issue-142349.fHK3v1.rst @@ -0,0 +1 @@ +Add :keyword:`lazy` to the list of support topic by :func:`help`. diff --git a/Misc/NEWS.d/next/Library/2026-05-15-19-52-41.gh-issue-149891.BJUIGB.rst b/Misc/NEWS.d/next/Library/2026-05-15-19-52-41.gh-issue-149891.BJUIGB.rst new file mode 100644 index 000000000000000..f8bc28659533af8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-15-19-52-41.gh-issue-149891.BJUIGB.rst @@ -0,0 +1 @@ +Add support for more encoding aliases `officially registered in IANA <https://www.iana.org/assignments/character-sets/character-sets.xhtml>`__. diff --git a/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst b/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst new file mode 100644 index 000000000000000..113bd1a802f7990 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-16-21-08-33.gh-issue-149921.I1yNML.rst @@ -0,0 +1,2 @@ +Fix reference leaks in error paths of the :mod:`!_interpchannels` and +:mod:`!_interpqueues` extension modules. diff --git a/Misc/NEWS.d/next/Library/2026-05-16-22-00-00.gh-issue-146406.10e0da.rst b/Misc/NEWS.d/next/Library/2026-05-16-22-00-00.gh-issue-146406.10e0da.rst new file mode 100644 index 000000000000000..8f65fecd85dbf4c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-16-22-00-00.gh-issue-146406.10e0da.rst @@ -0,0 +1,3 @@ +Add cross-language hints for ``.clear()`` on :class:`tuple`, +:class:`frozenset`, and :class:`frozendict`, suggesting the mutable +counterpart. Follow-up to :gh:`146406`. diff --git a/Misc/NEWS.d/next/Library/2026-05-17-02-25-56.gh-issue-149571.LNyuWJ.rst b/Misc/NEWS.d/next/Library/2026-05-17-02-25-56.gh-issue-149571.LNyuWJ.rst new file mode 100644 index 000000000000000..2b71d9cf2200be4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-17-02-25-56.gh-issue-149571.LNyuWJ.rst @@ -0,0 +1,2 @@ +Fix the C implementation of :meth:`xml.etree.ElementTree.Element.itertext`: +it no longer emits text for comments and processing instructions. diff --git a/Misc/NEWS.d/next/Library/2026-05-17-12-37-59.gh-issue-53144.c5tr1p.rst b/Misc/NEWS.d/next/Library/2026-05-17-12-37-59.gh-issue-53144.c5tr1p.rst new file mode 100644 index 000000000000000..283a5ba44d1f19f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-17-12-37-59.gh-issue-53144.c5tr1p.rst @@ -0,0 +1,2 @@ +The :mod:`email` package now supports all aliases of Python codecs and uses +MIME/IANA names for all IANA registered charsets. diff --git a/Misc/NEWS.d/next/Library/2026-05-17-22-37-02.gh-issue-88726.BAoL6j.rst b/Misc/NEWS.d/next/Library/2026-05-17-22-37-02.gh-issue-88726.BAoL6j.rst new file mode 100644 index 000000000000000..ba9058d79c9873a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-17-22-37-02.gh-issue-88726.BAoL6j.rst @@ -0,0 +1,2 @@ +The :mod:`email` package now uses standard MIME charset names "gb2312" and +"big5" instead of non-standard names "eucgb2312_cn" and "big5_tw". diff --git a/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst b/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst new file mode 100644 index 000000000000000..a8e412b578da378 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-07-44-46.gh-issue-149995.vvtFHn.rst @@ -0,0 +1 @@ +Update various docstrings in :mod:`typing`. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-12-42-31.gh-issue-149816.F98iME.rst b/Misc/NEWS.d/next/Library/2026-05-18-12-42-31.gh-issue-149816.F98iME.rst new file mode 100644 index 000000000000000..21e3ae0df57621c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-12-42-31.gh-issue-149816.F98iME.rst @@ -0,0 +1,2 @@ +Fix a potential use after free condition in :func:`pickle.dumps` in free-threaded +mode when serializing lists. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-13-43-06.gh-issue-79413.OpTXbV.rst b/Misc/NEWS.d/next/Library/2026-05-18-13-43-06.gh-issue-79413.OpTXbV.rst new file mode 100644 index 000000000000000..8d62fda8b2b67f4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-13-43-06.gh-issue-79413.OpTXbV.rst @@ -0,0 +1,3 @@ +Update :func:`dataclasses.make_dataclass` to add a *qualname* parameter. The +*qualname* parameter will be used to set the :attr:`!__qualname__` of the +created ``dataclass``. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-17-17-20.gh-issue-149189.a8IooK.rst b/Misc/NEWS.d/next/Library/2026-05-18-17-17-20.gh-issue-149189.a8IooK.rst new file mode 100644 index 000000000000000..bad027f2c71c6fd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-17-17-20.gh-issue-149189.a8IooK.rst @@ -0,0 +1 @@ +Revert the changes to :mod:`pprint` defaults. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-22-45-54.gh-issue-149816.T68vc_.rst b/Misc/NEWS.d/next/Library/2026-05-18-22-45-54.gh-issue-149816.T68vc_.rst new file mode 100644 index 000000000000000..9996cc7ec0e8664 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-22-45-54.gh-issue-149816.T68vc_.rst @@ -0,0 +1 @@ +Fix race condition in :attr:`ssl.SSLContext.sni_callback` diff --git a/Misc/NEWS.d/next/Library/2026-05-19-12-00-00.gh-issue-149980.Kx7p2A.rst b/Misc/NEWS.d/next/Library/2026-05-19-12-00-00.gh-issue-149980.Kx7p2A.rst new file mode 100644 index 000000000000000..b056f6a3cda25d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-19-12-00-00.gh-issue-149980.Kx7p2A.rst @@ -0,0 +1,3 @@ +Fix :mod:`tarfile` so that GNU long-name directory entries have all +trailing slashes stripped from their names, matching the behavior for +short-name entries. Previously, only a single trailing slash was removed. diff --git a/Misc/NEWS.d/next/Library/2026-05-19-19-00-49.gh-issue-84353.ZU5zaQ.rst b/Misc/NEWS.d/next/Library/2026-05-19-19-00-49.gh-issue-84353.ZU5zaQ.rst new file mode 100644 index 000000000000000..84fb12e2abd81a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-19-19-00-49.gh-issue-84353.ZU5zaQ.rst @@ -0,0 +1,5 @@ +Preserve non-UTF-8 encoded filenames when appending to a +:class:`zipfile.ZipFile`. Previously, non-ASCII names stored in a legacy +encoding (without the UTF-8 flag bit set) could be corrupted when the +central directory was rewritten: they were decoded as cp437 and then +re-stored as UTF-8. diff --git a/Misc/NEWS.d/next/Library/2026-05-21-11-25-58.gh-issue-150175.8H4Caz.rst b/Misc/NEWS.d/next/Library/2026-05-21-11-25-58.gh-issue-150175.8H4Caz.rst new file mode 100644 index 000000000000000..80fc80d4d50a636 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-21-11-25-58.gh-issue-150175.8H4Caz.rst @@ -0,0 +1,3 @@ +Fix race condition in :class:`unittest.mock.ThreadingMock` where +concurrent calls could lose increments to ``call_count`` and other +attributes due to a missing lock in ``_increment_mock_call``. diff --git a/Misc/NEWS.d/next/Library/2026-05-21-20-47-45.gh-issue-150157.ZvmO-bQZ.rst b/Misc/NEWS.d/next/Library/2026-05-21-20-47-45.gh-issue-150157.ZvmO-bQZ.rst new file mode 100644 index 000000000000000..3a12e26cf736f79 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-21-20-47-45.gh-issue-150157.ZvmO-bQZ.rst @@ -0,0 +1,3 @@ +Fix a crash in free-threaded builds that occurs when pickling by name +objects without a ``__module__`` attribute while :data:`sys.modules` +is concurrently being modified. diff --git a/Misc/NEWS.d/next/Library/2026-05-22-15-30-00.gh-issue-132372.YP4a6x.rst b/Misc/NEWS.d/next/Library/2026-05-22-15-30-00.gh-issue-132372.YP4a6x.rst new file mode 100644 index 000000000000000..bcd96e88eac1bfd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-22-15-30-00.gh-issue-132372.YP4a6x.rst @@ -0,0 +1,2 @@ +Speed up :func:`logging.config.fileConfig` and +:func:`logging.config.dictConfig` when handling many existing loggers. diff --git a/Misc/NEWS.d/next/Library/2026-05-25-07-22-05.gh-issue-150372.9hLqhe.rst b/Misc/NEWS.d/next/Library/2026-05-25-07-22-05.gh-issue-150372.9hLqhe.rst new file mode 100644 index 000000000000000..7b83bd8fe73f11d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-25-07-22-05.gh-issue-150372.9hLqhe.rst @@ -0,0 +1,2 @@ +:mod:`readline`: Fix a potential crash during tab completion caused by an +out-of-memory error during module initialization. diff --git a/Misc/NEWS.d/next/Library/2026-05-25-17-00-00.gh-issue-150406.jF3g63.rst b/Misc/NEWS.d/next/Library/2026-05-25-17-00-00.gh-issue-150406.jF3g63.rst new file mode 100644 index 000000000000000..230e961abd3f588 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-25-17-00-00.gh-issue-150406.jF3g63.rst @@ -0,0 +1,3 @@ +Fix a possible crash occurring during :mod:`socket` module initialization +when the system is out of memory on platforms without a reentrant +``gethostbyname``. diff --git a/Misc/NEWS.d/next/Library/2026-05-27-11-18-36.gh-issue-150228.pNPiO-.rst b/Misc/NEWS.d/next/Library/2026-05-27-11-18-36.gh-issue-150228.pNPiO-.rst new file mode 100644 index 000000000000000..8c03989e90b2401 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-27-11-18-36.gh-issue-150228.pNPiO-.rst @@ -0,0 +1,11 @@ +The new :class:`site.StartupState` class lets callers batch-process +:pep:`829` startup configuration files across multiple site directories +before any startup code runs, with public +:meth:`~site.StartupState.addsitedir`, +:meth:`~site.StartupState.addusersitepackages`, +:meth:`~site.StartupState.addsitepackages`, and +:meth:`~site.StartupState.process` methods. The signature of +:func:`site.addsitedir` is unchanged from Python 3.14. The +:data:`!defer_processing_start_files` argument and the +``process_startup_files()`` function added earlier in the 3.15 cycle have +been removed; use :class:`!site.StartupState` instead. diff --git a/Misc/NEWS.d/next/Library/2026-05-27-15-51-29.gh-issue-150534.3Nan6S.rst b/Misc/NEWS.d/next/Library/2026-05-27-15-51-29.gh-issue-150534.3Nan6S.rst new file mode 100644 index 000000000000000..2ead9990513015f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-27-15-51-29.gh-issue-150534.3Nan6S.rst @@ -0,0 +1,6 @@ +Added trigonometric functions that work in units of half turns, rather than +radians. The new functions :func:`math.acospi`, :func:`math.asinpi`, +:func:`math.atanpi`, and :func:`math.atan2pi` return half-turn angles. The +new functions :func:`math.cospi`, :func:`math.sinpi`, and :func:`math.tanpi` +take half-turn angle arguments. These functions are recommended by IEEE +754-2019 and standardized in C23. diff --git a/Misc/NEWS.d/next/Library/2026-05-27-23-47-31.gh-issue-148932.Y1xmvA.rst b/Misc/NEWS.d/next/Library/2026-05-27-23-47-31.gh-issue-148932.Y1xmvA.rst new file mode 100644 index 000000000000000..a0b7a9740cd518d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-27-23-47-31.gh-issue-148932.Y1xmvA.rst @@ -0,0 +1 @@ +Fix ``profiling.sampling`` on Windows virtual environments to resolve the actual Python PID from a virtual environment shim. diff --git a/Misc/NEWS.d/next/Library/2026-05-28-23-38-46.gh-issue-150478.qEr0E-.rst b/Misc/NEWS.d/next/Library/2026-05-28-23-38-46.gh-issue-150478.qEr0E-.rst new file mode 100644 index 000000000000000..0ceae5e313977a4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-28-23-38-46.gh-issue-150478.qEr0E-.rst @@ -0,0 +1,3 @@ +Add ``show_jit`` option to ``dis.dis`` to show JIT entry points in the +bytecode. This is useful for visualizing where JIT traces are entered from +the interpreter. diff --git a/Misc/NEWS.d/next/Library/2026-05-31-10-00-00.gh-issue-150942.Tk9aRef.rst b/Misc/NEWS.d/next/Library/2026-05-31-10-00-00.gh-issue-150942.Tk9aRef.rst new file mode 100644 index 000000000000000..ba0de8fd434d303 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-31-10-00-00.gh-issue-150942.Tk9aRef.rst @@ -0,0 +1,3 @@ +Speed up :func:`json.loads` decoding of arrays and objects by storing +parsed values into the result list/dict without an extra reference-count +round-trip (using the internal reference-stealing append/insert helpers). diff --git a/Misc/NEWS.d/next/Library/2026-05-31-12-00-00.gh-issue-150942.Re7Ref.rst b/Misc/NEWS.d/next/Library/2026-05-31-12-00-00.gh-issue-150942.Re7Ref.rst new file mode 100644 index 000000000000000..63967108b1e0b3b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-31-12-00-00.gh-issue-150942.Re7Ref.rst @@ -0,0 +1,3 @@ +Speed up :func:`re.findall`, :func:`re.sub` and :func:`re.subn` by appending +result items to the output list without an extra reference-count round-trip +(using the internal reference-stealing list append helper). diff --git a/Misc/NEWS.d/next/Library/2026-05-31-17-47-30.gh-issue-150685.EBB2mU.rst b/Misc/NEWS.d/next/Library/2026-05-31-17-47-30.gh-issue-150685.EBB2mU.rst new file mode 100644 index 000000000000000..eb7f31112d004c8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-31-17-47-30.gh-issue-150685.EBB2mU.rst @@ -0,0 +1 @@ +Update bundled pip to 26.1.2 diff --git a/Misc/NEWS.d/next/Library/2026-06-01-08-12-34.gh-issue-150717.LVRJXH.rst b/Misc/NEWS.d/next/Library/2026-06-01-08-12-34.gh-issue-150717.LVRJXH.rst new file mode 100644 index 000000000000000..da7171fcc72ce41 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-01-08-12-34.gh-issue-150717.LVRJXH.rst @@ -0,0 +1,2 @@ +Avoid an unnecessary per-call memory allocation when matching :mod:`re` +patterns that have no capturing groups. Patch by Bernát Gábor. diff --git a/Misc/NEWS.d/next/Library/2026-06-02-14-21-46.gh-issue-150750.SVS2o0.rst b/Misc/NEWS.d/next/Library/2026-06-02-14-21-46.gh-issue-150750.SVS2o0.rst new file mode 100644 index 000000000000000..bda500383e7cda3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-02-14-21-46.gh-issue-150750.SVS2o0.rst @@ -0,0 +1 @@ +Fix a race condition in :meth:`collections.deque.index` with free-threading. diff --git a/Misc/NEWS.d/next/Library/2026-06-02-15-44-57.gh-issue-150817.lpFCh0.rst b/Misc/NEWS.d/next/Library/2026-06-02-15-44-57.gh-issue-150817.lpFCh0.rst new file mode 100644 index 000000000000000..42532bb0222cf4e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-02-15-44-57.gh-issue-150817.lpFCh0.rst @@ -0,0 +1,2 @@ +Speed up the ``|``, ``&`` and ``^`` operations on :class:`enum.Flag` members. +Patch by Bernát Gábor. diff --git a/Misc/NEWS.d/next/Library/2026-06-02-15-44-58.gh-issue-150818.Orcefu.rst b/Misc/NEWS.d/next/Library/2026-06-02-15-44-58.gh-issue-150818.Orcefu.rst new file mode 100644 index 000000000000000..3bb16001cc0bb41 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-02-15-44-58.gh-issue-150818.Orcefu.rst @@ -0,0 +1,3 @@ +Speed up :func:`logging.getLogger` with a lock-free fast path that returns an +already-registered logger without acquiring the logging lock. Patch by Bernát +Gábor. diff --git a/Misc/NEWS.d/next/Library/2026-06-03-13-51-29.gh-issue-150662.ELT8Vg.rst b/Misc/NEWS.d/next/Library/2026-06-03-13-51-29.gh-issue-150662.ELT8Vg.rst new file mode 100644 index 000000000000000..42ed6ad7cd3c65f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-03-13-51-29.gh-issue-150662.ELT8Vg.rst @@ -0,0 +1,4 @@ +Fix the ``--gecko`` collector in :mod:`profiling.sampling` that kept every +sample in memory. It now writes sample and marker data to temporary files +and reads them back, ultimately building the output file at the end. Patch +by Pablo Galindo and Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-06-03-21-59-11.gh-issue-150886.r2c25g.rst b/Misc/NEWS.d/next/Library/2026-06-03-21-59-11.gh-issue-150886.r2c25g.rst new file mode 100644 index 000000000000000..b220b63cfdb4ab5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-03-21-59-11.gh-issue-150886.r2c25g.rst @@ -0,0 +1,4 @@ +Remove the private, undocumented function +``importlib._bootstrap._object_name()``. It had no caller after +``load_module()`` and its deprecation warnings were removed from +:mod:`importlib`. diff --git a/Misc/NEWS.d/next/Library/2026-06-04-06-50-06.gh-issue-150898.1LkLA3.rst b/Misc/NEWS.d/next/Library/2026-06-04-06-50-06.gh-issue-150898.1LkLA3.rst new file mode 100644 index 000000000000000..85328c43368e198 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-04-06-50-06.gh-issue-150898.1LkLA3.rst @@ -0,0 +1 @@ +Unconditionally assume :attr:`ssl.SSLContext.keylog_filename` exists. diff --git a/Misc/NEWS.d/next/Library/2026-06-04-10-44-36.gh-issue-150889.UYNLR_.rst b/Misc/NEWS.d/next/Library/2026-06-04-10-44-36.gh-issue-150889.UYNLR_.rst new file mode 100644 index 000000000000000..a5cc1da758637c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-04-10-44-36.gh-issue-150889.UYNLR_.rst @@ -0,0 +1 @@ +Speed up :func:`unicodedata.normalize` for the NFC and NFKC forms of non-ASCII text up to a factor 2. diff --git a/Misc/NEWS.d/next/Library/2026-06-04-18-22-56.gh-issue-143008.z5tw-J.rst b/Misc/NEWS.d/next/Library/2026-06-04-18-22-56.gh-issue-143008.z5tw-J.rst new file mode 100644 index 000000000000000..e99bc39c45f9b8f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-04-18-22-56.gh-issue-143008.z5tw-J.rst @@ -0,0 +1 @@ +Fix race conditions when re-initializing a :class:`io.TextIOWrapper` object. diff --git a/Misc/NEWS.d/next/Library/2026-06-04-19-24-13.gh-issue-140006.TD8HKl.rst b/Misc/NEWS.d/next/Library/2026-06-04-19-24-13.gh-issue-140006.TD8HKl.rst new file mode 100644 index 000000000000000..6d8d66b29b19902 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-04-19-24-13.gh-issue-140006.TD8HKl.rst @@ -0,0 +1,4 @@ +The :mod:`venv` ``activate.fish`` script now calls fish builtins through +``builtin`` so a user function that shadows ``.``/``source``, ``echo``, +``printf``, ``set_color``, or ``functions`` can no longer hijack the virtual +environment prompt or break exit-status reporting. diff --git a/Misc/NEWS.d/next/Library/2026-06-04-21-49-18.gh-issue-150913.EmptyBl.rst b/Misc/NEWS.d/next/Library/2026-06-04-21-49-18.gh-issue-150913.EmptyBl.rst new file mode 100644 index 000000000000000..f95a6ee6ee15bf7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-04-21-49-18.gh-issue-150913.EmptyBl.rst @@ -0,0 +1,3 @@ +Fix :class:`sqlite3.Blob` slice assignment to raise +:exc:`TypeError` and :exc:`IndexError` for type and size mismatches +respectively, even when the target slice is empty. diff --git a/Misc/NEWS.d/next/Library/2026-06-04-23-10-31.gh-issue-62825.BtG_yQ.rst b/Misc/NEWS.d/next/Library/2026-06-04-23-10-31.gh-issue-62825.BtG_yQ.rst new file mode 100644 index 000000000000000..95a4fb1c61d4c30 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-04-23-10-31.gh-issue-62825.BtG_yQ.rst @@ -0,0 +1,2 @@ +Encodings "KS_C_5601-1987", "KS X 1001", etc are now aliases of "CP949" +instead of "EUC-KR". diff --git a/Misc/NEWS.d/next/Library/2026-06-06-06-29-17.gh-issue-150994.I2119M.rst b/Misc/NEWS.d/next/Library/2026-06-06-06-29-17.gh-issue-150994.I2119M.rst new file mode 100644 index 000000000000000..c4a610c24806b0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-06-06-29-17.gh-issue-150994.I2119M.rst @@ -0,0 +1 @@ +Make the type annotations in the private ``_colorize`` module resolvable. diff --git a/Misc/NEWS.d/next/Library/2026-06-06-15-20-54.gh-issue-151021.J4qk2A.rst b/Misc/NEWS.d/next/Library/2026-06-06-15-20-54.gh-issue-151021.J4qk2A.rst new file mode 100644 index 000000000000000..0617fa068c844d6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-06-15-20-54.gh-issue-151021.J4qk2A.rst @@ -0,0 +1,3 @@ +Fix :meth:`mmap.mmap.find` and :meth:`~mmap.mmap.rfind` to return ``-1`` +when searching for an empty subsequence with a start position past the end +of the mapping. diff --git a/Misc/NEWS.d/next/Library/2026-06-07-13-47-05.gh-issue-151042.NMdzF2.rst b/Misc/NEWS.d/next/Library/2026-06-07-13-47-05.gh-issue-151042.NMdzF2.rst new file mode 100644 index 000000000000000..a24a15c0cc267f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-07-13-47-05.gh-issue-151042.NMdzF2.rst @@ -0,0 +1 @@ +Fixed venv exporting mixed path styles to PATH on Windows inside fish (via Cygwin, MinGW or MSYS2) diff --git a/Misc/NEWS.d/next/Library/2026-06-07-17-29-33.gh-issue-151039.AZ0qBn.rst b/Misc/NEWS.d/next/Library/2026-06-07-17-29-33.gh-issue-151039.AZ0qBn.rst new file mode 100644 index 000000000000000..1e99567f5550579 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-07-17-29-33.gh-issue-151039.AZ0qBn.rst @@ -0,0 +1 @@ +Fix a crash when static :mod:`datetime` types outlive the ``_datetime`` module. diff --git a/Misc/NEWS.d/next/Library/2026-06-08-15-04-35.gh-issue-150285.-Tj94n.rst b/Misc/NEWS.d/next/Library/2026-06-08-15-04-35.gh-issue-150285.-Tj94n.rst new file mode 100644 index 000000000000000..60b1f3e0a71f5bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-08-15-04-35.gh-issue-150285.-Tj94n.rst @@ -0,0 +1 @@ +:mod:`pydoc` now wraps long single-line summary in text output. diff --git a/Misc/NEWS.d/next/Library/2026-06-09-12-00-00.gh-issue-150771.K7mNx2.rst b/Misc/NEWS.d/next/Library/2026-06-09-12-00-00.gh-issue-150771.K7mNx2.rst new file mode 100644 index 000000000000000..6535e5c48bf0360 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-09-12-00-00.gh-issue-150771.K7mNx2.rst @@ -0,0 +1,4 @@ +Fix :mod:`email` messages created with ``shift_jis`` or ``euc-jp`` charsets. +``set_content()`` now stores the payload using the output charset +(``iso-2022-jp``) so printing the message no longer raises +:exc:`UnicodeEncodeError`. diff --git a/Misc/NEWS.d/next/Library/2026-06-09-14-49-17.gh-issue-80384.ttRAja.rst b/Misc/NEWS.d/next/Library/2026-06-09-14-49-17.gh-issue-80384.ttRAja.rst new file mode 100644 index 000000000000000..53d4d513ba757de --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-09-14-49-17.gh-issue-80384.ttRAja.rst @@ -0,0 +1,2 @@ +:func:`weakref.ref` and :func:`weakref.proxy` now raise :exc:`TypeError` if +the *callback* argument is not callable or ``None``. diff --git a/Misc/NEWS.d/next/Library/2026-06-09-19-43-24.gh-issue-151179.hl_6Z0.rst b/Misc/NEWS.d/next/Library/2026-06-09-19-43-24.gh-issue-151179.hl_6Z0.rst new file mode 100644 index 000000000000000..904edf3a8c70908 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-09-19-43-24.gh-issue-151179.hl_6Z0.rst @@ -0,0 +1,3 @@ +Fix a pidfd leak in ``_PidfdChildWatcher`` on Linux: the watcher no +longer leaks the process file descriptor when ``waitpid()`` fails with an +error other than :exc:`ChildProcessError`. diff --git a/Misc/NEWS.d/next/Library/2026-06-10-00-00-02.gh-issue-109940.Cx1099.rst b/Misc/NEWS.d/next/Library/2026-06-10-00-00-02.gh-issue-109940.Cx1099.rst new file mode 100644 index 000000000000000..130dc780b612864 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-10-00-00-02.gh-issue-109940.Cx1099.rst @@ -0,0 +1,2 @@ +Fix Windows :mod:`venv` activation in ``cmd.exe`` to respect +``VIRTUAL_ENV_DISABLE_PROMPT``. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-00-00-00.gh-issue-151295.NQYUzW.rst b/Misc/NEWS.d/next/Library/2026-06-11-00-00-00.gh-issue-151295.NQYUzW.rst new file mode 100644 index 000000000000000..e9012f023ff7f77 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-00-00-00.gh-issue-151295.NQYUzW.rst @@ -0,0 +1,4 @@ +Fixed a crash (use-after-free) in :meth:`bytes.join` and +:meth:`bytearray.join` that could occur if an item's +:meth:`~object.__buffer__` concurrently mutates the sequence being joined. +The mutation is now reported as a :exc:`RuntimeError` instead. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-06-56-31.gh-issue-150994.gd1wVw.rst b/Misc/NEWS.d/next/Library/2026-06-11-06-56-31.gh-issue-150994.gd1wVw.rst new file mode 100644 index 000000000000000..005a1d99b766dce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-06-56-31.gh-issue-150994.gd1wVw.rst @@ -0,0 +1 @@ +Make type annotations in the private ``_colorize`` module resolvable. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-11-52-23.gh-issue-149671.6Rpr5r.rst b/Misc/NEWS.d/next/Library/2026-06-11-11-52-23.gh-issue-149671.6Rpr5r.rst new file mode 100644 index 000000000000000..5c08828e5fd77ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-11-52-23.gh-issue-149671.6Rpr5r.rst @@ -0,0 +1,3 @@ +Restore compatibility with setuptools ``-nspkg.pth`` files in the :mod:`site` +module. Inject ``sitedir`` variable in the frame which executes pth code. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-16-25-38.gh-issue-151126.bh_Usy.rst b/Misc/NEWS.d/next/Library/2026-06-11-16-25-38.gh-issue-151126.bh_Usy.rst new file mode 100644 index 000000000000000..25149057aa7d092 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-16-25-38.gh-issue-151126.bh_Usy.rst @@ -0,0 +1,2 @@ +Fix a crash when :exc:`MemoryError` in :func:`!os._path_splitroot` +was not set properly. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-19-46-16.gh-issue-150285.wuhAsL.rst b/Misc/NEWS.d/next/Library/2026-06-11-19-46-16.gh-issue-150285.wuhAsL.rst new file mode 100644 index 000000000000000..3f19150488f3131 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-19-46-16.gh-issue-150285.wuhAsL.rst @@ -0,0 +1,2 @@ +:mod:`pydoc` now uses all available space (80 columns) for formatting reprs +of module and class data, but ensure that they do not overflow. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-21-43-24.gh-issue-151337.JSVV18.rst b/Misc/NEWS.d/next/Library/2026-06-11-21-43-24.gh-issue-151337.JSVV18.rst new file mode 100644 index 000000000000000..0344eee9471d292 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-21-43-24.gh-issue-151337.JSVV18.rst @@ -0,0 +1 @@ +Avoid possible memory leak in ``tkinter.c`` on Windows. diff --git a/Misc/NEWS.d/next/Library/2026-06-12-00-04-34.gh-issue-151126.aHaBYq.rst b/Misc/NEWS.d/next/Library/2026-06-12-00-04-34.gh-issue-151126.aHaBYq.rst new file mode 100644 index 000000000000000..20ef69d5de5ac55 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-12-00-04-34.gh-issue-151126.aHaBYq.rst @@ -0,0 +1,2 @@ +Fix crash on unset :exc:`MemoryError` on allocation failure in +:func:`ctypes.get_errno`. diff --git a/Misc/NEWS.d/next/Library/2026-06-12-00-17-29.gh-issue-151128.-LYO3a.rst b/Misc/NEWS.d/next/Library/2026-06-12-00-17-29.gh-issue-151128.-LYO3a.rst new file mode 100644 index 000000000000000..6e760686af2b01f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-12-00-17-29.gh-issue-151128.-LYO3a.rst @@ -0,0 +1,3 @@ +Cross-language keyword suggestions are now shown for :exc:`SyntaxError` messages. +For example, ``switch x:`` suggests ``match``, ``delete x`` suggests ``del``, +``function f():`` suggests ``def``. Contributed by Zang Langyan. diff --git a/Misc/NEWS.d/next/Library/2026-06-12-07-20-08.gh-issue-151390.CmYN9EeJ.rst b/Misc/NEWS.d/next/Library/2026-06-12-07-20-08.gh-issue-151390.CmYN9EeJ.rst new file mode 100644 index 000000000000000..ff8de30599c6ad5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-12-07-20-08.gh-issue-151390.CmYN9EeJ.rst @@ -0,0 +1 @@ +Colorize ``match`` in the :term:`REPL` when followed by a unary ``+`` or ``-`` operator. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2026-06-12-22-46-31.gh-issue-151403.DalZWh.rst b/Misc/NEWS.d/next/Library/2026-06-12-22-46-31.gh-issue-151403.DalZWh.rst new file mode 100644 index 000000000000000..ca779ed684e7616 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-12-22-46-31.gh-issue-151403.DalZWh.rst @@ -0,0 +1,3 @@ +Fixed a crash in :class:`subprocess.Popen` (and ``_posixsubprocess.fork_exec``) +when an ``argv`` item's :meth:`~os.PathLike.__fspath__` concurrently mutates the +``args`` sequence being converted. diff --git a/Misc/NEWS.d/next/Library/2026-06-13-04-11-00.gh-issue-151426.f2V67e.rst b/Misc/NEWS.d/next/Library/2026-06-13-04-11-00.gh-issue-151426.f2V67e.rst new file mode 100644 index 000000000000000..428302e5f847f36 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-13-04-11-00.gh-issue-151426.f2V67e.rst @@ -0,0 +1,4 @@ +Fix impossible stack traces (callers and callees cross called, orphans and +incorrect lines) in the Tachyon profiler when caching frames, by snapshotting +the stack chunks before walking the frame chain on a cache miss. Patch by +Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-06-13-11-57-48.gh-issue-151436.UEDowO.rst b/Misc/NEWS.d/next/Library/2026-06-13-11-57-48.gh-issue-151436.UEDowO.rst new file mode 100644 index 000000000000000..1d1aadbf57be485 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-13-11-57-48.gh-issue-151436.UEDowO.rst @@ -0,0 +1,4 @@ +Fix skewed stack trackes in the Tachyon profiler when caching is enabled and +when generators and coroutines are profiled, by updating +``tstate->last_profiled_frame`` at every frame-removal site. The issue resulted +in total erasure of some callers. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Library/2026-06-15-07-42-12.gh-issue-151485.NiYQPZ.rst b/Misc/NEWS.d/next/Library/2026-06-15-07-42-12.gh-issue-151485.NiYQPZ.rst new file mode 100644 index 000000000000000..134499107ab2b78 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-15-07-42-12.gh-issue-151485.NiYQPZ.rst @@ -0,0 +1 @@ +Fix command quoting in :exc:`subprocess.CalledProcessError`. Contributed by Benjy Wiener. diff --git a/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst b/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst new file mode 100644 index 000000000000000..a4c03c9d71d7618 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst @@ -0,0 +1,4 @@ +Opening a :mod:`tarfile` archive no longer attempts to pre-allocate a huge +buffer when a crafted or truncated member claims an oversized extended header +(a GNU long name/link or a pax header). The extended header is now read in +bounded chunks, so its size field can no longer trigger memory exhaustion. diff --git a/Misc/NEWS.d/next/Library/2026-06-18-23-59-46.gh-issue-151596.5ma144.rst b/Misc/NEWS.d/next/Library/2026-06-18-23-59-46.gh-issue-151596.5ma144.rst new file mode 100644 index 000000000000000..17ec1341142e93d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-18-23-59-46.gh-issue-151596.5ma144.rst @@ -0,0 +1 @@ +Add missing ``size`` positional argument to the pure-Python implementation of :meth:`io.TextIOBase.readline`. diff --git a/Misc/NEWS.d/next/Library/2026-06-19-07-26-20.gh-issue-151695.IBDlkN.rst b/Misc/NEWS.d/next/Library/2026-06-19-07-26-20.gh-issue-151695.IBDlkN.rst new file mode 100644 index 000000000000000..f44cb6b93071656 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-19-07-26-20.gh-issue-151695.IBDlkN.rst @@ -0,0 +1,4 @@ +Fix a use-after-free in the :mod:`curses` module. The encoding of the initial +screen, used by :func:`curses.unctrl` and :func:`curses.ungetch` to encode +non-ASCII characters, is now kept as a private copy instead of a borrowed +pointer to a window object that may be deallocated. diff --git a/Misc/NEWS.d/next/Library/2026-06-19-12-00-00.gh-issue-151674.FCYTvs.rst b/Misc/NEWS.d/next/Library/2026-06-19-12-00-00.gh-issue-151674.FCYTvs.rst new file mode 100644 index 000000000000000..84e21d4cbc4bd72 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-19-12-00-00.gh-issue-151674.FCYTvs.rst @@ -0,0 +1,3 @@ +Add the :meth:`~tkinter.Text.edit_canundo` and :meth:`~tkinter.Text.edit_canredo` +methods of :class:`!tkinter.Text`, wrapping the Tk ``edit canundo`` and +``edit canredo`` subcommands. diff --git a/Misc/NEWS.d/next/Library/2026-06-19-12-00-01.gh-issue-151675.acLD7L.rst b/Misc/NEWS.d/next/Library/2026-06-19-12-00-01.gh-issue-151675.acLD7L.rst new file mode 100644 index 000000000000000..92bd69de6bdc0ec --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-19-12-00-01.gh-issue-151675.acLD7L.rst @@ -0,0 +1,3 @@ +Add the :meth:`~tkinter.Text.sync` and :meth:`~tkinter.Text.pendingsync` +methods of :class:`!tkinter.Text`, wrapping the Tk ``sync`` and +``pendingsync`` subcommands. diff --git a/Misc/NEWS.d/next/Library/2026-06-19-12-19-30.gh-issue-90092.yBVc0C.rst b/Misc/NEWS.d/next/Library/2026-06-19-12-19-30.gh-issue-90092.yBVc0C.rst new file mode 100644 index 000000000000000..41e0ce80ee8a069 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-19-12-19-30.gh-issue-90092.yBVc0C.rst @@ -0,0 +1,4 @@ +Add support for multiple terminals to the :mod:`curses` module: the new +functions :func:`curses.newterm`, :func:`curses.set_term` and +:func:`curses.new_prescr`, the corresponding :ref:`screen +<curses-screen-objects>` object, and the :meth:`curses.window.use` method. diff --git a/Misc/NEWS.d/next/Library/2026-06-19-18-40-00.gh-issue-151744.Kp7mNq.rst b/Misc/NEWS.d/next/Library/2026-06-19-18-40-00.gh-issue-151744.Kp7mNq.rst new file mode 100644 index 000000000000000..24328a8f9f93427 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-19-18-40-00.gh-issue-151744.Kp7mNq.rst @@ -0,0 +1 @@ +Add :func:`curses.nofilter`, which undoes the effect of :func:`curses.filter`. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-02-14-55.gh-issue-151757.TP9A2x.rst b/Misc/NEWS.d/next/Library/2026-06-20-02-14-55.gh-issue-151757.TP9A2x.rst new file mode 100644 index 000000000000000..34db9698738c2b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-02-14-55.gh-issue-151757.TP9A2x.rst @@ -0,0 +1,7 @@ +The :mod:`curses` character-cell window methods now accept a full character +cell -- a spacing character optionally followed by combining characters -- in +addition to a single integer or byte character. Add the wide-character read +methods :meth:`curses.window.get_wstr` and :meth:`curses.window.in_wstr`, and +the functions :func:`curses.erasewchar`, :func:`curses.killwchar` and +:func:`curses.wunctrl`. These features are only available when built against +the wide-character ncursesw library. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-11-11-22.gh-issue-151774.O6nrvs.rst b/Misc/NEWS.d/next/Library/2026-06-20-11-11-22.gh-issue-151774.O6nrvs.rst new file mode 100644 index 000000000000000..668b01ebfb8ad79 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-11-11-22.gh-issue-151774.O6nrvs.rst @@ -0,0 +1,5 @@ +Add the :mod:`curses` functions :func:`curses.alloc_pair`, +:func:`curses.find_pair`, :func:`curses.free_pair` and +:func:`curses.reset_color_pairs` for dynamic color-pair management. They are +only available when Python is built against a wide-character version of the +underlying curses library with extended-color support. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-12-10-02.gh-issue-151776.BtsXIF.rst b/Misc/NEWS.d/next/Library/2026-06-20-12-10-02.gh-issue-151776.BtsXIF.rst new file mode 100644 index 000000000000000..3eabc4a6bea82b5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-12-10-02.gh-issue-151776.BtsXIF.rst @@ -0,0 +1,12 @@ +Add :mod:`curses` functions and window methods that report state which could +previously only be set: the window methods :meth:`~curses.window.is_cleared`, +:meth:`~curses.window.is_idcok`, :meth:`~curses.window.is_idlok`, +:meth:`~curses.window.is_immedok`, :meth:`~curses.window.is_keypad`, +:meth:`~curses.window.is_leaveok`, :meth:`~curses.window.is_nodelay`, +:meth:`~curses.window.is_notimeout`, :meth:`~curses.window.is_pad`, +:meth:`~curses.window.is_scrollok`, :meth:`~curses.window.is_subwin`, +:meth:`~curses.window.is_syncok`, :meth:`~curses.window.getdelay`, +:meth:`~curses.window.getparent` and :meth:`~curses.window.getscrreg`, and the +functions :func:`curses.is_cbreak`, :func:`curses.is_echo`, +:func:`curses.is_nl` and :func:`curses.is_raw`. They are only available when +built against an ncurses with ``NCURSES_EXT_FUNCS``. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-151770.dtiso0.rst b/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-151770.dtiso0.rst new file mode 100644 index 000000000000000..10b3db8efa42b0f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-151770.dtiso0.rst @@ -0,0 +1,3 @@ +Fix :meth:`datetime.datetime.fromisoformat` raising :exc:`AssertionError` +instead of :exc:`ValueError` for an out-of-range month combined with a +``24:00`` time. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-75666.Kt9xQ2.rst b/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-75666.Kt9xQ2.rst new file mode 100644 index 000000000000000..d2b2b066837bb1f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-75666.Kt9xQ2.rst @@ -0,0 +1,2 @@ +Fix a reference leak in :mod:`tkinter`: the Tcl commands created for event +callbacks are now deleted when a binding is replaced or unbound. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-21-15-13.gh-issue-151814.OIbgsO.rst b/Misc/NEWS.d/next/Library/2026-06-20-21-15-13.gh-issue-151814.OIbgsO.rst new file mode 100644 index 000000000000000..1365fb4d8edb1d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-21-15-13.gh-issue-151814.OIbgsO.rst @@ -0,0 +1,2 @@ +Fix unbounded memory growth in :class:`io.TextIOWrapper` when repeatedly +writing an empty string. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-22-55-22.gh-issue-59396.kT9wPq.rst b/Misc/NEWS.d/next/Library/2026-06-20-22-55-22.gh-issue-59396.kT9wPq.rst new file mode 100644 index 000000000000000..550620a9c8276cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-22-55-22.gh-issue-59396.kT9wPq.rst @@ -0,0 +1,22 @@ +The :mod:`tkinter.simpledialog` dialogs were modernized to match the look and +feel of the native Tk dialogs. +:class:`!tkinter.simpledialog.SimpleDialog` and the +:func:`~tkinter.simpledialog.askinteger`, +:func:`~tkinter.simpledialog.askfloat` and +:func:`~tkinter.simpledialog.askstring` dialogs are now built from +the themed :mod:`tkinter.ttk` widgets instead of the classic :mod:`tkinter` +widgets; the :class:`!tkinter.simpledialog.Dialog` base class still defaults to +the classic widgets for compatibility. Both :class:`!Dialog` and +:class:`!SimpleDialog` gained a *use_ttk* +parameter that selects between the classic Tk widgets and the themed ttk +widgets. :class:`!SimpleDialog` also gained *bitmap* and *detail* parameters, +draws the standard icons with themed images in the ttk version, and accepts +mappings of button options as *buttons* entries, where an ``underline`` option +adds an :kbd:`Alt` key accelerator. +The font and wrap length of the message and the detail message are taken from +the Tk option database and can be overridden by the application. +The dialogs also follow the keyboard conventions of the Tk message box: the +default ring follows the keyboard focus, the Return key activates the focused +button, and the :class:`!Dialog` OK and Cancel buttons gained :kbd:`Alt` key +accelerators. +Several bugs were also fixed. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-00-46-57.gh-issue-151874.8VtTrq.rst b/Misc/NEWS.d/next/Library/2026-06-22-00-46-57.gh-issue-151874.8VtTrq.rst new file mode 100644 index 000000000000000..50a33f5be809871 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-00-46-57.gh-issue-151874.8VtTrq.rst @@ -0,0 +1,3 @@ +Add the :mod:`tkinter` methods :meth:`!Misc.winfo_isdark`, +:meth:`!Wm.wm_iconbadge` and :meth:`!Wm.wm_stackorder`, wrapping the +``winfo isdark``, ``wm iconbadge`` and ``wm stackorder`` Tk commands. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-00-55-07.gh-issue-151876.sVe0xx.rst b/Misc/NEWS.d/next/Library/2026-06-22-00-55-07.gh-issue-151876.sVe0xx.rst new file mode 100644 index 000000000000000..9dc5f2327cf4155 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-00-55-07.gh-issue-151876.sVe0xx.rst @@ -0,0 +1,2 @@ +Add the :class:`tkinter.Canvas` methods :meth:`!rchars` and :meth:`!rotate`, +wrapping the ``rchars`` and ``rotate`` Tk canvas commands. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-01-02-33.gh-issue-151878.IFIA5C.rst b/Misc/NEWS.d/next/Library/2026-06-22-01-02-33.gh-issue-151878.IFIA5C.rst new file mode 100644 index 000000000000000..af0d2c25136c452 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-01-02-33.gh-issue-151878.IFIA5C.rst @@ -0,0 +1,3 @@ +Add the :meth:`!validate` method to the :class:`tkinter.Entry` and +:class:`tkinter.Spinbox` widgets, forcing an evaluation of the validation +command. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-01-39-38.gh-issue-151881.ShACSZ.rst b/Misc/NEWS.d/next/Library/2026-06-22-01-39-38.gh-issue-151881.ShACSZ.rst new file mode 100644 index 000000000000000..9ae939438e826f6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-01-39-38.gh-issue-151881.ShACSZ.rst @@ -0,0 +1,4 @@ +Add the :meth:`tkinter.Menu.postcascade` method and the +:meth:`!tkinter.Misc.tk_scaling` and :meth:`!tkinter.Misc.tk_inactive` +methods, wrapping the ``postcascade``, ``tk scaling`` and ``tk inactive`` +Tk commands. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-01-51-06.gh-issue-151886.MsLlNz.rst b/Misc/NEWS.d/next/Library/2026-06-22-01-51-06.gh-issue-151886.MsLlNz.rst new file mode 100644 index 000000000000000..7a454ab95f371f1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-01-51-06.gh-issue-151886.MsLlNz.rst @@ -0,0 +1,4 @@ +Add the :meth:`!tkinter.Misc.tk_appname`, +:meth:`!tkinter.Misc.tk_useinputmethods` and :meth:`!tkinter.Misc.tk_caret` +methods, wrapping the ``tk appname``, ``tk useinputmethods`` and ``tk caret`` +Tk commands. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-01-57-55.gh-issue-151888.hO7mxi.rst b/Misc/NEWS.d/next/Library/2026-06-22-01-57-55.gh-issue-151888.hO7mxi.rst new file mode 100644 index 000000000000000..8a2d5ed765d234e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-01-57-55.gh-issue-151888.hO7mxi.rst @@ -0,0 +1,2 @@ +Add the :meth:`!tkinter.PhotoImage.redither` method, wrapping the photo image +``redither`` Tk command. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-02-29-02.gh-issue-151890.gT6JwI.rst b/Misc/NEWS.d/next/Library/2026-06-22-02-29-02.gh-issue-151890.gT6JwI.rst new file mode 100644 index 000000000000000..3e064aeeba8131c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-02-29-02.gh-issue-151890.gT6JwI.rst @@ -0,0 +1,6 @@ +Add the *format* parameter to the :meth:`tkinter.PhotoImage.put` method, the +*metadata* parameter to the :meth:`~tkinter.PhotoImage.put`, +:meth:`~tkinter.PhotoImage.read`, :meth:`~tkinter.PhotoImage.write` and +:meth:`~tkinter.PhotoImage.data` methods, and the *withalpha* parameter to the +:meth:`~tkinter.PhotoImage.get` method. The *metadata* and *withalpha* +parameters require Tcl/Tk 9.0 or newer. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-10-30-22.gh-issue-151910.ayi8xK.rst b/Misc/NEWS.d/next/Library/2026-06-22-10-30-22.gh-issue-151910.ayi8xK.rst new file mode 100644 index 000000000000000..3e5647cbf835490 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-10-30-22.gh-issue-151910.ayi8xK.rst @@ -0,0 +1,6 @@ +Added :class:`tkinter.ttk.Treeview` methods wrapping the enhanced +``ttk::treeview`` widget commands added in Tk 9.1 (and the ``detached`` query +added in Tk 9.0): item navigation and queries, opening, hiding, sorting and +searching of items, and cell focus, selection and tagging. The +:meth:`~tkinter.ttk.Treeview.expand` and :meth:`~tkinter.ttk.Treeview.collapse` +methods (without recursion) also work on Tk older than 9.1. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-14-30-00.gh-issue-151920.Th3mSt.rst b/Misc/NEWS.d/next/Library/2026-06-22-14-30-00.gh-issue-151920.Th3mSt.rst new file mode 100644 index 000000000000000..2c4a069a794b10e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-14-30-00.gh-issue-151920.Th3mSt.rst @@ -0,0 +1,3 @@ +Add the :meth:`ttk.Style.theme_styles <tkinter.ttk.Style.theme_styles>` +method, wrapping the Tk ``ttk::style theme styles`` subcommand, which +returns the list of styles defined in a theme. diff --git a/Misc/NEWS.d/next/Library/2026-06-22-18-01-00.gh-issue-151955.6u5iwm.rst b/Misc/NEWS.d/next/Library/2026-06-22-18-01-00.gh-issue-151955.6u5iwm.rst new file mode 100644 index 000000000000000..4cb5f04025327c4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-22-18-01-00.gh-issue-151955.6u5iwm.rst @@ -0,0 +1,2 @@ +Allow more types to be used in the ``bound`` argument to +:class:`typing.ParamSpec` and :class:`typing.TypeVarTuple`. diff --git a/Misc/NEWS.d/next/Library/2026-06-23-12-00-00.gh-issue-101284.Ab3Xy9.rst b/Misc/NEWS.d/next/Library/2026-06-23-12-00-00.gh-issue-101284.Ab3Xy9.rst new file mode 100644 index 000000000000000..9afcf8dcf545ed5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-23-12-00-00.gh-issue-101284.Ab3Xy9.rst @@ -0,0 +1,2 @@ +:class:`tkinter.OptionMenu` now accepts arbitrary :class:`!tkinter.Menubutton` +options as keyword arguments and uses them to override its default appearance. diff --git a/Misc/NEWS.d/next/Library/2026-06-23-13-27-14.gh-issue-126219.kOfv2g.rst b/Misc/NEWS.d/next/Library/2026-06-23-13-27-14.gh-issue-126219.kOfv2g.rst new file mode 100644 index 000000000000000..8cc8c64c719471d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-23-13-27-14.gh-issue-126219.kOfv2g.rst @@ -0,0 +1,3 @@ +Fixed a crash in :class:`tkinter.Tk` when *className* contains a non-BMP +character and tkinter is built against Tcl/Tk 8.x. Such a name is now +rejected with a :exc:`ValueError`. diff --git a/Misc/NEWS.d/next/Library/2026-06-23-17-14-04.gh-issue-127802.nGSxo6.rst b/Misc/NEWS.d/next/Library/2026-06-23-17-14-04.gh-issue-127802.nGSxo6.rst new file mode 100644 index 000000000000000..9484c9dbd23b2fa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-23-17-14-04.gh-issue-127802.nGSxo6.rst @@ -0,0 +1,3 @@ +The deprecated :class:`tkinter.Variable` methods :meth:`!trace_variable`, +:meth:`!trace`, :meth:`!trace_vdelete` and :meth:`!trace_vinfo` are now +scheduled for removal in Python 3.17. diff --git a/Misc/NEWS.d/next/Library/2026-06-23-22-15-00.gh-issue-152033.Ct1Egy.rst b/Misc/NEWS.d/next/Library/2026-06-23-22-15-00.gh-issue-152033.Ct1Egy.rst new file mode 100644 index 000000000000000..7c54a6c16c1a145 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-23-22-15-00.gh-issue-152033.Ct1Egy.rst @@ -0,0 +1,5 @@ +Optimize matching of character class escapes (``\d``, ``\D``, ``\s``, +``\S``, ``\w`` and ``\W``) that occur outside a character set: they are now +compiled to a single ``CATEGORY`` opcode instead of being wrapped in an +``IN`` block. This speeds up patterns such as ``\d+`` and reduces the size +of the compiled byte code. diff --git a/Misc/NEWS.d/next/Library/2026-06-24-10-30-00.gh-issue-152056.Qk7mZ2.rst b/Misc/NEWS.d/next/Library/2026-06-24-10-30-00.gh-issue-152056.Qk7mZ2.rst new file mode 100644 index 000000000000000..6e71d720cd19be2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-10-30-00.gh-issue-152056.Qk7mZ2.rst @@ -0,0 +1,5 @@ +Optimize matching of a character set that contains a single character +category, such as ``[\d]`` or ``[^\s]``: it is now compiled to a single +``CATEGORY`` opcode, the same as the corresponding ``\d`` or ``\S`` escape, +instead of being wrapped in an ``IN`` block. This speeds up matching and +reduces the size of the compiled byte code. Patch by Pieter Eendebak. diff --git a/Misc/NEWS.d/next/Library/2026-06-24-10-46-35.gh-issue-152060.f2asrt.rst b/Misc/NEWS.d/next/Library/2026-06-24-10-46-35.gh-issue-152060.f2asrt.rst new file mode 100644 index 000000000000000..6088a6b80829bb4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-10-46-35.gh-issue-152060.f2asrt.rst @@ -0,0 +1,3 @@ +Fix :meth:`datetime.datetime.fromisoformat` raising :exc:`AssertionError` +instead of :exc:`ValueError` for some malformed strings in the pure-Python +implementation, matching the C implementation. diff --git a/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152079.f1tzus.rst b/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152079.f1tzus.rst new file mode 100644 index 000000000000000..492d00724f6a46e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152079.f1tzus.rst @@ -0,0 +1,3 @@ +Fix :meth:`datetime.datetime.fromisoformat` in the C implementation dropping +the sub-second part of a UTC offset whose whole-second part is zero, matching +the pure-Python implementation. diff --git a/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152100.Set0ps.rst b/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152100.Set0ps.rst new file mode 100644 index 000000000000000..848740ed7a56d31 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152100.Set0ps.rst @@ -0,0 +1,3 @@ +Support set operations and nested sets in regular expression character +classes, as described in Unicode Technical Standard #18: set difference +(``[A--B]``), intersection (``[A&&B]``) and union (``[A||B]``). diff --git a/Misc/NEWS.d/next/Library/2026-06-24-22-16-35.gh-issue-105895.hRkuEw.rst b/Misc/NEWS.d/next/Library/2026-06-24-22-16-35.gh-issue-105895.hRkuEw.rst new file mode 100644 index 000000000000000..c69e6fa7d14f4dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-22-16-35.gh-issue-105895.hRkuEw.rst @@ -0,0 +1,2 @@ +Add :keyword:`match` and :keyword:`case` to the list of supported topics by +:func:`help`. diff --git a/Misc/NEWS.d/next/Library/2026-06-25-01-00-43.gh-issue-151763.wWeHBe.rst b/Misc/NEWS.d/next/Library/2026-06-25-01-00-43.gh-issue-151763.wWeHBe.rst new file mode 100644 index 000000000000000..2f5e84027ad31bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-25-01-00-43.gh-issue-151763.wWeHBe.rst @@ -0,0 +1,2 @@ +Fix crash in :func:`!_interpqueues.create` whe :exc:`MemoryError` +happens on queue creation. diff --git a/Misc/NEWS.d/next/Library/2026-06-30-13-00-00.gh-issue-151842.OOM31g.rst b/Misc/NEWS.d/next/Library/2026-06-30-13-00-00.gh-issue-151842.OOM31g.rst new file mode 100644 index 000000000000000..8a2ecf3c40032f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-30-13-00-00.gh-issue-151842.OOM31g.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`!_interpreters.capture_exception` when +:exc:`MemoryError` happens. Patch by Amrutha Modela. diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst new file mode 100644 index 000000000000000..ca554997e5c3963 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst @@ -0,0 +1,2 @@ +Normalize all line endings (CR, CRLF, and LF) to LF+TAB when writing +multi-line configparser values. diff --git a/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst b/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst deleted file mode 100644 index 4993633b8ebebb2..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-03-20-09-29-42.gh-issue-146211.PQVbs7.rst +++ /dev/null @@ -1,2 +0,0 @@ -Reject CR/LF characters in tunnel request headers for the -HTTPConnection.set_tunnel() method. diff --git a/Misc/NEWS.d/next/Security/2026-03-25-00-51-03.gh-issue-146333.LqdL__bn.rst b/Misc/NEWS.d/next/Security/2026-03-25-00-51-03.gh-issue-146333.LqdL__bn.rst deleted file mode 100644 index 96d86ecc0a0fb3e..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-03-25-00-51-03.gh-issue-146333.LqdL__bn.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix quadratic backtracking in :class:`configparser.RawConfigParser` option -parsing regexes (``OPTCRE`` and ``OPTCRE_NV``). A crafted configuration line -with many whitespace characters could cause excessive CPU usage. diff --git a/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst b/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst deleted file mode 100644 index 98e65549d79016f..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-03-29-12-51-33.gh-issue-146581.4vZfB0.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix vulnerability in :func:`shutil.unpack_archive` for ZIP files on Windows -which allowed to write files outside of the destination tree if the patch in -the archive contains a Windows drive prefix. Now such invalid paths will be -skipped. Files containing ".." in the name (like "foo..bar") are no longer -skipped. diff --git a/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst b/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst deleted file mode 100644 index 45cdeebe1b6d64a..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-03-31-09-15-51.gh-issue-148169.EZJzz2.rst +++ /dev/null @@ -1,2 +0,0 @@ -A bypass in :mod:`webbrowser` allowed URLs prefixed with ``%action`` to pass -the dash-prefix safety check. diff --git a/Misc/NEWS.d/next/Security/2026-04-06-13-55-00.gh-issue-148178.Rs7kLm.rst b/Misc/NEWS.d/next/Security/2026-04-06-13-55-00.gh-issue-148178.Rs7kLm.rst deleted file mode 100644 index ed138a54a859dec..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-04-06-13-55-00.gh-issue-148178.Rs7kLm.rst +++ /dev/null @@ -1,2 +0,0 @@ -Hardened :mod:`!_remote_debugging` by validating remote debug offset tables -before using them to size memory reads or interpret remote layouts. diff --git a/Misc/NEWS.d/next/Security/2026-04-08-14-25-47.gh-issue-148252.IEp9Rt.rst b/Misc/NEWS.d/next/Security/2026-04-08-14-25-47.gh-issue-148252.IEp9Rt.rst deleted file mode 100644 index adc2c9287149db3..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-04-08-14-25-47.gh-issue-148252.IEp9Rt.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed stack depth calculation in :mod:`!_remote_debugging` when decoding -certain ``.pyb`` inputs on 32-bit builds. Issue originally identified and -diagnosed by Tristan Madani (@TristanInSec on GitHub). diff --git a/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst b/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst deleted file mode 100644 index 9502189ab199c15..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-04-10-16-28-21.gh-issue-148395.kfzm0G.rst +++ /dev/null @@ -1,5 +0,0 @@ -Fix a dangling input pointer in :class:`lzma.LZMADecompressor`, -:class:`bz2.BZ2Decompressor`, and internal :class:`!zlib._ZlibDecompressor` -when memory allocation fails with :exc:`MemoryError`, which could let a -subsequent :meth:`!decompress` call read or write through a stale pointer to -the already-released caller buffer. diff --git a/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst b/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst deleted file mode 100644 index 0b5cf85fedfba10..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-04-20-15-31-37.gh-issue-148808._Z8JL0.rst +++ /dev/null @@ -1,3 +0,0 @@ -Added buffer boundary check when using ``nbytes`` parameter with -:meth:`!asyncio.AbstractEventLoop.sock_recvfrom_into`. Only -relevant for Windows and the :class:`asyncio.ProactorEventLoop`. diff --git a/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst b/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst deleted file mode 100644 index d7d376737e4ad11..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-04-21-13-46-30.gh-issue-90309.srvj9q.rst +++ /dev/null @@ -1,3 +0,0 @@ -Base64-encode values when embedding cookies to JavaScript using the -:meth:`http.cookies.BaseCookie.js_output` method to avoid injection -and escaping. diff --git a/Misc/NEWS.d/next/Security/2026-04-24-23-15-42.gh-issue-148252.8BLmzd.rst b/Misc/NEWS.d/next/Security/2026-04-24-23-15-42.gh-issue-148252.8BLmzd.rst deleted file mode 100644 index 531ea2348ffdefa..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-04-24-23-15-42.gh-issue-148252.8BLmzd.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed string table and sample record bounds checks in :mod:`!_remote_debugging` -when decoding certain ``.pyb`` inputs on 32-bit builds. Patch by Maurycy -Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Security/2026-04-26-19-30-45.gh-issue-149018.a9SqWb.rst b/Misc/NEWS.d/next/Security/2026-04-26-19-30-45.gh-issue-149018.a9SqWb.rst new file mode 100644 index 000000000000000..d1b5b368684e6a5 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-26-19-30-45.gh-issue-149018.a9SqWb.rst @@ -0,0 +1,3 @@ +Improved protection against XML hash-flooding attacks in +:mod:`xml.parsers.expat` and :mod:`xml.etree.ElementTree` when Python is +compiled with libExpat 2.8.0 or later. diff --git a/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst b/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst new file mode 100644 index 000000000000000..4ed22b58f7405f5 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-04-27-16-36-11.gh-issue-149079.vKl-LM.rst @@ -0,0 +1,5 @@ +Fix a potential denial of service in :func:`unicodedata.normalize`. The +canonical ordering step of Unicode normalization used a quadratic-time insertion +sort for reordering combining characters, which could be exploited with +crafted input containing many combining characters in non-canonical order. +Replaced with a linear-time counting sort for long runs. diff --git a/Misc/NEWS.d/next/Security/2026-05-02-15-38-03.gh-issue-149254.0HOL0j.rst b/Misc/NEWS.d/next/Security/2026-05-02-15-38-03.gh-issue-149254.0HOL0j.rst deleted file mode 100644 index f3cf924db25b4cb..000000000000000 --- a/Misc/NEWS.d/next/Security/2026-05-02-15-38-03.gh-issue-149254.0HOL0j.rst +++ /dev/null @@ -1 +0,0 @@ -Update Android and iOS installer to use OpenSSL 3.5.6. diff --git a/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst new file mode 100644 index 000000000000000..7c69edb683cf80a --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-03-21-00-00.gh-issue-149486.tarflt.rst @@ -0,0 +1,5 @@ +:func:`tarfile.data_filter` now validates link targets using the same +normalised value that is written to disk, strips trailing separators from +the member name when resolving a symlink's directory, and rejects link +members that would replace the destination directory itself. This closes +several path-traversal bypasses of the ``data`` extraction filter. diff --git a/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst b/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst new file mode 100644 index 000000000000000..48e718b95ebe3ae --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-08-02-18-54.gh-issue-149474.ujQ-mu.rst @@ -0,0 +1,3 @@ +Fix the binary writer in :mod:`profiling.sampling` not firing the audit +(:pep:`578`) when creating the output file. The writer and the reader now +accept any path-like object. Patch by Maurycy Pawłowski-Wieroński. diff --git a/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst new file mode 100644 index 000000000000000..21a79c3e0e7db74 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-10-18-05-32.gh-issue-87451.XkKB6M.rst @@ -0,0 +1,6 @@ +The :mod:`ftplib` module's undocumented ``ftpcp`` function no longer trusts +the IPv4 address value returned from the source server in response to the +``PASV`` command by default, completing the fix for CVE-2021-4189. As with +:class:`ftplib.FTP`, the former behavior can be re-enabled by setting the +``trust_server_pasv_ipv4_address`` attribute on the source :class:`ftplib.FTP` +instance to ``True``. Thanks to Qi Deng at Aurascape AI for the report. diff --git a/Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst b/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst similarity index 58% rename from Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst rename to Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst index 6aa7efb68a1981e..3c8671b9a5adc43 100644 --- a/Misc/NEWS.d/next/Security/2026-04-26-17-49-58.gh-issue-149017.EiVFPo.rst +++ b/Misc/NEWS.d/next/Security/2026-05-11-21-15-07.gh-issue-149698.OudOcW.rst @@ -1 +1,2 @@ -Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.0. +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.1 +for the fix for :cve:`2026-45186`. diff --git a/Misc/NEWS.d/next/Security/2026-05-18-17-46-00.gh-issue-149835.EebFlk.rst b/Misc/NEWS.d/next/Security/2026-05-18-17-46-00.gh-issue-149835.EebFlk.rst new file mode 100644 index 000000000000000..20cab736552486d --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-18-17-46-00.gh-issue-149835.EebFlk.rst @@ -0,0 +1,3 @@ +:func:`shutil.move` now resolves symlinks via :func:`os.path.realpath` +when checking whether the destination is inside the source directory, +preventing a symlink-based bypass of that guard. diff --git a/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst b/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst new file mode 100644 index 000000000000000..a37d86cf423f820 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-30-09-36-20.gh-issue-150599.nlHqU-.rst @@ -0,0 +1,3 @@ +Fix a possible stack buffer overflow in :mod:`bz2` when a +:class:`bz2.BZ2Decompressor` is reused after a decompression error. +The decompressor now becomes unusable after libbz2 reports an error. diff --git a/Misc/NEWS.d/next/Security/2026-06-09-10-23-57.gh-issue-151159.91GpWQ.rst b/Misc/NEWS.d/next/Security/2026-06-09-10-23-57.gh-issue-151159.91GpWQ.rst new file mode 100644 index 000000000000000..735164c1a65ec33 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-09-10-23-57.gh-issue-151159.91GpWQ.rst @@ -0,0 +1 @@ +Update Android and iOS installers to use OpenSSL 3.5.7. diff --git a/Misc/NEWS.d/next/Security/2026-06-09-23-38-08.gh-issue-151159.ds-9f8.rst b/Misc/NEWS.d/next/Security/2026-06-09-23-38-08.gh-issue-151159.ds-9f8.rst new file mode 100644 index 000000000000000..d9251a93b40b2cc --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-09-23-38-08.gh-issue-151159.ds-9f8.rst @@ -0,0 +1 @@ +Update macOS installer to use OpenSSL 3.5.7. diff --git a/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst b/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst new file mode 100644 index 000000000000000..74459d5680e21a3 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst @@ -0,0 +1,3 @@ +Fixed an vulnerability in the :mod:`tarfile` ``data`` and ``tar`` extraction +filters where crafted archives could create a symlink pointing outside the +destination directory. This was a bypass of :cve:`2025-4330`. diff --git a/Misc/NEWS.d/next/Security/2026-06-16-14-58-02.gh-issue-151544._bexVy.rst b/Misc/NEWS.d/next/Security/2026-06-16-14-58-02.gh-issue-151544._bexVy.rst new file mode 100644 index 000000000000000..418e3b4b9677943 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-16-14-58-02.gh-issue-151544._bexVy.rst @@ -0,0 +1,4 @@ +:file:`Modules/Setup.local` is no longer used as a landmark to discover +whether Python is running in a source tree, as it could potentially affect +actual installs. The :file:`pybuilddir.txt` file is now the sole indicator +of running in a source tree. diff --git a/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst b/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst new file mode 100644 index 000000000000000..2123ab8e081b1d9 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst @@ -0,0 +1,2 @@ +In :mod:`tarfile`, seeking a stream now stops when end of the stream is +reached. diff --git a/Misc/NEWS.d/next/Tests/2026-04-21-12-33-14.gh-issue-148600.vnTb3t.rst b/Misc/NEWS.d/next/Tests/2026-04-21-12-33-14.gh-issue-148600.vnTb3t.rst deleted file mode 100644 index a7d4c0cc80a146c..000000000000000 --- a/Misc/NEWS.d/next/Tests/2026-04-21-12-33-14.gh-issue-148600.vnTb3t.rst +++ /dev/null @@ -1 +0,0 @@ -Add OpenSSL 4.0.0 support to test configurations. diff --git a/Misc/NEWS.d/next/Tests/2026-04-24-01-38-56.gh-issue-148853._uM4_Q.rst b/Misc/NEWS.d/next/Tests/2026-04-24-01-38-56.gh-issue-148853._uM4_Q.rst new file mode 100644 index 000000000000000..9d3fbc2590dc7a2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-04-24-01-38-56.gh-issue-148853._uM4_Q.rst @@ -0,0 +1,2 @@ +Fix tests failing on FreeBSD in test.support's +in_systemd_nspawn_sync_suppressed() due to unreadable /run directory. diff --git a/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst b/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst new file mode 100644 index 000000000000000..e86a9130ff9bfb6 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-05-13-14-53-23.gh-issue-149776.orqgsn.rst @@ -0,0 +1,2 @@ +Fix test_socket on Linux kernel 7.1 and newer: skip UDP Lite tests if it's +not supported. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst b/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst new file mode 100644 index 000000000000000..a140bf921972ed9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-05-22-16-41-45.gh-issue-150114.UdMikH.rst @@ -0,0 +1,2 @@ +On Linux, regrtest now logs the total memory usage of all Python processes. +Read the private memory in ``/proc/pid/smaps``. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2026-05-25-15-39-53.gh-issue-150387.yzZ7jq.rst b/Misc/NEWS.d/next/Tests/2026-05-25-15-39-53.gh-issue-150387.yzZ7jq.rst new file mode 100644 index 000000000000000..663a357a1792042 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-05-25-15-39-53.gh-issue-150387.yzZ7jq.rst @@ -0,0 +1,5 @@ +Fix hang in +``test.test_profiling.test_sampling_profiler.test_live_collector_ui.TestLiveModeErrors.test_run_failed_script_live`` +on slow buildbots. The test now always queues a final ``q`` keystroke so the +live TUI loop exits even when the profiler collects enough samples to enter +the post-finished input loop. diff --git a/Misc/NEWS.d/next/Tests/2026-06-06-16-22-00.gh-issue-150966.7N9x5Q.rst b/Misc/NEWS.d/next/Tests/2026-06-06-16-22-00.gh-issue-150966.7N9x5Q.rst new file mode 100644 index 000000000000000..3bbb471163d64e1 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-06-06-16-22-00.gh-issue-150966.7N9x5Q.rst @@ -0,0 +1,2 @@ +Avoid prematurely terminating failing live sampling profiler test targets, +which made stderr assertions flaky on ASAN buildbots. diff --git a/Misc/NEWS.d/next/Tests/2026-06-09-11-52-52.gh-issue-151130.1vslPH.rst b/Misc/NEWS.d/next/Tests/2026-06-09-11-52-52.gh-issue-151130.1vslPH.rst new file mode 100644 index 000000000000000..0333e66446ce161 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-06-09-11-52-52.gh-issue-151130.1vslPH.rst @@ -0,0 +1 @@ +Add more tests for ``PyWeakref_*`` C API. diff --git a/Misc/NEWS.d/next/Tools-Demos/2024-06-17-13-52-04.gh-issue-111501.syFLAV.rst b/Misc/NEWS.d/next/Tools-Demos/2024-06-17-13-52-04.gh-issue-111501.syFLAV.rst new file mode 100644 index 000000000000000..2989d4b81b1dd39 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2024-06-17-13-52-04.gh-issue-111501.syFLAV.rst @@ -0,0 +1 @@ +PS1 is no longer exported by venv activate script diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-05-22-18-51-09.gh-issue-150258.dh8GVK.rst b/Misc/NEWS.d/next/Tools-Demos/2026-05-22-18-51-09.gh-issue-150258.dh8GVK.rst new file mode 100644 index 000000000000000..02cad6c4f53d928 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-05-22-18-51-09.gh-issue-150258.dh8GVK.rst @@ -0,0 +1 @@ +Update the tooltip on the Tachyon flame graph to show both absolute and relative percentages. diff --git a/Misc/NEWS.d/next/Windows/2026-03-27-22-06-10.gh-issue-146458.fYj0UQ.rst b/Misc/NEWS.d/next/Windows/2026-03-27-22-06-10.gh-issue-146458.fYj0UQ.rst deleted file mode 100644 index 178c04c657ecbbd..000000000000000 --- a/Misc/NEWS.d/next/Windows/2026-03-27-22-06-10.gh-issue-146458.fYj0UQ.rst +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect REPL height and width tracking on console window resize on Windows. diff --git a/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst b/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst new file mode 100644 index 000000000000000..4afb8f737b692e8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-04-29-14-44-51.gh-issue-138489.234aj6.rst @@ -0,0 +1,4 @@ +Windows distributions now include a :file:`build-details.json` file (see +:pep:`739`). The legacy installer does not install it, but all other +distributions from python.org and all preset configurations in the +``PC\layout`` script will include one. diff --git a/Misc/NEWS.d/next/Windows/2026-05-03-13-55-51.gh-issue-149254.ENtMYD.rst b/Misc/NEWS.d/next/Windows/2026-05-03-13-55-51.gh-issue-149254.ENtMYD.rst deleted file mode 100644 index 946654700a10c56..000000000000000 --- a/Misc/NEWS.d/next/Windows/2026-05-03-13-55-51.gh-issue-149254.ENtMYD.rst +++ /dev/null @@ -1 +0,0 @@ -Updated bundled version of OpenSSL to 3.5.6. diff --git a/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst b/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst new file mode 100644 index 000000000000000..9a57536f1dc96b6 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst @@ -0,0 +1 @@ +Updated Windows builds to use Tcl/Tk 9.0.3. diff --git a/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst b/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst new file mode 100644 index 000000000000000..64ca91a01f41afc --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-05-14-22-09-46.gh-issue-149786.UI-HZM.rst @@ -0,0 +1 @@ +Fixes virtual environment launchers on Windows free-threaded builds. diff --git a/Misc/NEWS.d/next/Windows/2026-06-04-18-53-18.gh-issue-150836.Wci7bZ.rst b/Misc/NEWS.d/next/Windows/2026-06-04-18-53-18.gh-issue-150836.Wci7bZ.rst new file mode 100644 index 000000000000000..6497b7927db7da3 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-06-04-18-53-18.gh-issue-150836.Wci7bZ.rst @@ -0,0 +1 @@ +Make installed tkinter work with Tcl/Tk 9 builds that embed the Tk script library in the Tk DLL on Windows. diff --git a/Misc/NEWS.d/next/Windows/2026-06-09-11-40-48.gh-issue-151159.JKVfme.rst b/Misc/NEWS.d/next/Windows/2026-06-09-11-40-48.gh-issue-151159.JKVfme.rst new file mode 100644 index 000000000000000..ad1be115db5ce8f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-06-09-11-40-48.gh-issue-151159.JKVfme.rst @@ -0,0 +1 @@ +Updated bundled version of OpenSSL to 3.5.7. diff --git a/Misc/NEWS.d/next/Windows/2026-06-09-11-55-41.gh-issue-151163.oizZYV.rst b/Misc/NEWS.d/next/Windows/2026-06-09-11-55-41.gh-issue-151163.oizZYV.rst new file mode 100644 index 000000000000000..580a87400862c52 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-06-09-11-55-41.gh-issue-151163.oizZYV.rst @@ -0,0 +1 @@ +Updated Windows builds to include SQLite version 3.53.2. diff --git a/Misc/NEWS.d/next/macOS/2026-05-01-20-12-33.gh-issue-149254.kXdWpS.rst b/Misc/NEWS.d/next/macOS/2026-05-01-20-12-33.gh-issue-149254.kXdWpS.rst deleted file mode 100644 index 278327c91f1baa4..000000000000000 --- a/Misc/NEWS.d/next/macOS/2026-05-01-20-12-33.gh-issue-149254.kXdWpS.rst +++ /dev/null @@ -1 +0,0 @@ -Update macOS installer to use OpenSSL 3.5.6. diff --git a/Misc/NEWS.d/next/macOS/2026-05-31-10-40-00.gh-issue-150644.zLWyjj.rst b/Misc/NEWS.d/next/macOS/2026-05-31-10-40-00.gh-issue-150644.zLWyjj.rst new file mode 100644 index 000000000000000..7452a7c765c0a86 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2026-05-31-10-40-00.gh-issue-150644.zLWyjj.rst @@ -0,0 +1,3 @@ +When system logging is enabled (with ``config.use_system_logger``, messages +are now tagged as public. This allows the macOS 26 system logger to view +messages without special configuration. diff --git a/Misc/NEWS.d/next/macOS/2026-06-09-11-52-35.gh-issue-151163.RlPXHq.rst b/Misc/NEWS.d/next/macOS/2026-06-09-11-52-35.gh-issue-151163.RlPXHq.rst new file mode 100644 index 000000000000000..7e9bf6f4587974e --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2026-06-09-11-52-35.gh-issue-151163.RlPXHq.rst @@ -0,0 +1 @@ +Updated macOS installer to include SQLite version 3.53.2. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 593fa01bf25ed1e..523d20259adaaa9 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,84 +70,84 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "cf01946f3a61ba45a08c1e35b223d41d23963e3df5ac98cbad6c8fa5a81070ca" + "checksumValue": "ca94e7c6c223d9caf77bb51aac5949186379608ea2a0cad3aa8bdf31856912e9" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.5.6.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.5.7.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.5.6:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.5.7:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.5.6" + "versionInfo": "3.5.7" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "fb5ab81f27612b0a7b4861ba655906c76dc85ee969e7a4905d2075aff931e8d0" + "checksumValue": "53f8711811090cc4d9ffc624c360f81e7b409763b145ab2e948998f1a0d6a612" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.50.4.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.53.2.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.50.4.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.53.2.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "sqlite", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.50.4.0" + "versionInfo": "3.53.2.0" }, { - "SPDXID": "SPDXRef-PACKAGE-tcl-core", + "SPDXID": "SPDXRef-PACKAGE-tcl", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "4c23f0dd3efcbe6f3a22c503a68d147617bb30c4f5290f1eb3eaacf0b460440b" + "checksumValue": "7a1d1f3a2b8f4484a9c2a027a157963c18f85a81785e85fcb5d1e3df6b6a4fd4" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-core-8.6.15.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-9.0.3.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", - "name": "tcl-core", + "name": "tcl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.15.0" + "versionInfo": "9.0.3.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "0ae56d39bca92865f338529557a1e56d110594184b6dc5a91339c5675751e264" + "checksumValue": "54fb59df12c489c6264f5b7d3d7444b150d1e3d6561fd59cdb11483440cec000" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-8.6.15.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-9.0.3.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:8.6.15.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8.6.15.0" + "versionInfo": "9.0.3.1" }, { "SPDXID": "SPDXRef-PACKAGE-xz", diff --git a/Misc/python.man b/Misc/python.man index a65fb98a697b503..5e4f3c3debe9358 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -1,4 +1,4 @@ -.TH PYTHON "1" +.TH PYTHON 1 .\" To view this file while editing, run it through groff: .\" groff -Tascii -man python.man | less @@ -162,7 +162,7 @@ compilation options). Ignore environment variables like PYTHONPATH and PYTHONHOME that modify the behavior of the interpreter. .TP -.B \-h ", " \-? ", "\-\-help +.BR \-h ", " \-? ", " \-\-help Prints the usage for the interpreter executable and exits. .TP .B "\-\-help\-env" @@ -171,7 +171,6 @@ Prints help about Python-specific environment variables and exits. .B "\-\-help\-xoptions" Prints help about implementation-specific \fB\-X\fP options and exits. .TP -.TP .B "\-\-help\-all" Prints complete usage information and exits. .TP @@ -243,7 +242,7 @@ twice, print a message for each file that is checked for when searching for a module. Also provides information on module cleanup at exit. .TP -.B \-V ", " \-\-version +.BR \-V ", " \-\-version Prints the Python version number of the executable and exits. When given twice, print more information about the build. @@ -255,23 +254,38 @@ to The simplest settings apply a particular action unconditionally to all warnings emitted by a process (even those that are otherwise ignored by default): - - -Wdefault # Warn once per call location - -Werror # Convert to exceptions - -Walways # Warn every time - -Wall # Same as -Walways - -Wmodule # Warn once per calling module - -Wonce # Warn once per Python process - -Wignore # Never warn - +.RS +.TP +.B \-Wdefault +Warn once per call location +.TP +.B \-Werror +Convert to exceptions +.TP +.B \-Walways +Warn every time +.TP +.B \-Wall +Same as \-Walways +.TP +.B \-Wmodule +Warn once per calling module +.TP +.B \-Wonce +Warn once per Python process +.TP +.B \-Wignore +Never warn +.RE +.IP The action names can be abbreviated as desired and the interpreter will resolve them to the appropriate action name. For example, .B \-Wi is the same as -.B \-Wignore . +.BR \-Wignore . The full form of argument is: -.IB action:message:category:module:lineno +.IB action : message : category : module : lineno Empty fields match all values; trailing empty fields may be omitted. For example @@ -457,7 +471,7 @@ is an empty string; if is used, .I sys.argv[0] contains the string -.I '\-c'. +.RI ' \-c '. Note that options interpreted by the Python interpreter itself are not placed in .IR sys.argv . @@ -531,12 +545,12 @@ the \fB\-d\fP option. If set to an integer, it is equivalent to specifying \fB\-d\fP multiple times. .IP PYTHONEXECUTABLE If this environment variable is set, -.IB sys.argv[0] +.I sys.argv[0] will be set to its value instead of the value got through the C runtime. Only works on Mac OS X. .IP PYTHONFAULTHANDLER If this environment variable is set to a non-empty string, -.IR faulthandler.enable() +.I faulthandler.enable() is called at startup: install a handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals to dump the Python traceback. .IP @@ -584,8 +598,8 @@ purpose is to allow repeatable hashing, such as for selftests for the interpreter itself, or to allow a cluster of python processes to share hash values. -The integer must be a decimal number in the range [0,4294967295]. Specifying -the value 0 will disable hash randomization. +The integer must be a decimal number in the range [0,4\|294\|967\|295]. +Specifying the value 0 will disable hash randomization. .IP PYTHONHOME Change the location of the standard Python libraries. By default, the libraries are searched in ${prefix}/lib/python<version> and @@ -607,16 +621,16 @@ This is equivalent to the \fB\-X int_max_str_digits=\fINUMBER\fR option. .IP PYTHONIOENCODING If this is set before running the interpreter, it overrides the encoding used for stdin/stdout/stderr, in the syntax -.IB encodingname ":" errorhandler +.IB encodingname : errorhandler The -.IB errorhandler +.I errorhandler part is optional and has the same meaning as in str.encode. For stderr, the -.IB errorhandler +.I errorhandler part is ignored; the handler will always be \'backslashreplace\'. .IP PYTHONMALLOC Set the Python memory allocators and/or install debug hooks. The available memory allocators are -.IR malloc +.I malloc and .IR pymalloc . The available debug hooks are @@ -626,7 +640,7 @@ and .IR pymalloc_debug . .IP When Python is compiled in debug mode, the default is -.IR pymalloc_debug +.I pymalloc_debug and the debug hooks are automatically used. Otherwise, the default is .IR pymalloc . .IP PYTHONMALLOCSTATS @@ -707,14 +721,14 @@ Python memory allocations using the tracemalloc module. .IP The value of the variable is the maximum number of frames stored in a traceback of a trace. For example, -.IB PYTHONTRACEMALLOC=1 +.I PYTHONTRACEMALLOC=1 stores only the most recent frame. .IP PYTHONUNBUFFERED If this is set to a non-empty string it is equivalent to specifying the \fB\-u\fP option. .IP PYTHONUSERBASE Defines the user base directory, which is used to compute the path of the user -.IR site\-packages +.I site\-packages directory and installation paths for .IR "python \-m pip install \-\-user" . .IP PYTHONUTF8 @@ -750,17 +764,17 @@ This is equivalent to the \fB\-X presite=\fImodule\fR option. .SH AUTHOR The Python Software Foundation: https://www.python.org/psf/ .SH INTERNET RESOURCES -Main website: https://www.python.org/ +Main website: https://www.python.org/ .br -Documentation: https://docs.python.org/ +Documentation: https://docs.python.org/ .br -Developer resources: https://devguide.python.org/ +Developer resources: https://devguide.python.org/ .br -Downloads: https://www.python.org/downloads/ +Downloads: https://www.python.org/downloads/ .br -Module repository: https://pypi.org/ +Module repository: https://pypi.org/ .br -Newsgroups: comp.lang.python, comp.lang.python.announce +Newsgroups: comp.lang.python, comp.lang.python.announce .SH LICENSING Python is distributed under an Open Source license. See the file "LICENSE" in the Python source distribution for information on terms & diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index aaeffd58e799ede..5791592dc77a84f 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "5343adc95840915b022b1d4524d0acb66b369ba2" + "checksumValue": "58101ef0951568acadd3117033bef084fea24cc1" }, { "algorithm": "SHA256", - "checksumValue": "1ec3bad08b6864c2c479e1fd941038c2dcd24c6d9a16400f4da54912d95aa321" + "checksumValue": "52d756026bf09befdb211c453e2009a646d6c6b519e6885e971b2550396619fb" } ], "fileName": "Modules/expat/expat.h" @@ -174,11 +174,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "cb0af01558ec7b6474d2bd0c9386380c82618e8f" + "checksumValue": "1dad2ab196cdbe37572674c465bd9187fdbe4495" }, { "algorithm": "SHA256", - "checksumValue": "6745a6b8cdd7344d4bd8f27f605363ed746e57ff02d4ebce3eb1806579cd030f" + "checksumValue": "740137e670d2f3b7269364ffb6f60064e6560091850c5d6f2c3bb1b8ca6e3dd1" } ], "fileName": "Modules/expat/xmlparse.c" @@ -608,11 +608,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0a0b7f3714167ad45ddf5a6a48d76f525a119c9c" + "checksumValue": "67a029736da5efe96f1b2d41b92f00934b955253" }, { "algorithm": "SHA256", - "checksumValue": "135d4afb4812468885c963c9c87a55ba5fae9181df4431af5fbad08588dda229" + "checksumValue": "0d80c2e3f5ff3b31ef0d435fc3196660df89ccf95bdcbb6545dc2f9df07bec9f" } ], "fileName": "Modules/_hacl/Lib_Memzero0.c" @@ -916,11 +916,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "63e47cc290c4ec887dca708000876ac37ee75ba0" + "checksumValue": "8d6a23f4e932cace0c67896bdda46cb24bf5e0ea" }, { "algorithm": "SHA256", - "checksumValue": "d09a6196d65c2645974100eb922002bd387d3ae13f2653780f82ed97a79af635" + "checksumValue": "25decbcadf1fef51f473daa4975a3754480fc903b03068bf95f149fb5349456e" } ], "fileName": "Modules/_hacl/libintvector.h" @@ -994,734 +994,6 @@ } ], "fileName": "Lib/ctypes/macholib/framework.py" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-README.txt", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "bda6e0bd6121f7069b420bdc0bc7c49414d948d1" - }, - { - "algorithm": "SHA256", - "checksumValue": "89926cd0fe6cfb33a2b5b7416c101e9b5d42b0d639d348e0871acf6ffc8258a3" - } - ], - "fileName": "Modules/_decimal/libmpdec/README.txt" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "33757ce2ec0c93c1b5e03c45a495563a00e498ae" - }, - { - "algorithm": "SHA256", - "checksumValue": "ad498362c31a5b99ab19fce320ac540cf14c5c4ec09478f0ad3858da1428113d" - } - ], - "fileName": "Modules/_decimal/libmpdec/basearith.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "bf03919412c068e6969e7ac48850f91bfcd3b2b1" - }, - { - "algorithm": "SHA256", - "checksumValue": "2eaac88a71b9bcf3144396c12dcfeced573e0e550a0050d75b9ed3903248596d" - } - ], - "fileName": "Modules/_decimal/libmpdec/basearith.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bench.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "c925b7f26754ae182aaa461d51802e8b6a2bb5e9" - }, - { - "algorithm": "SHA256", - "checksumValue": "007e38542ec8d9d8805fe243b5390d79211b9360e2797a20079e833e68ad9e45" - } - ], - "fileName": "Modules/_decimal/libmpdec/bench.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bench-full.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "cb22686269685a53a17afdea9ed984714e399d9d" - }, - { - "algorithm": "SHA256", - "checksumValue": "1b9e892d4b268deea835ec8906f20a1e5d25e037b2e698edcd34315613f3608c" - } - ], - "fileName": "Modules/_decimal/libmpdec/bench_full.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-bits.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "fc91c2450cdf1e785d1347411662294c3945eb27" - }, - { - "algorithm": "SHA256", - "checksumValue": "ce7741e58ea761a24250c0bfa10058cec8c4fd220dca70a41de3927a2e4f5376" - } - ], - "fileName": "Modules/_decimal/libmpdec/bits.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "7187c18916b0a546ec19b4fc4bec43d0d9fb5fc2" - }, - { - "algorithm": "SHA256", - "checksumValue": "cd430b8657cf8a616916e02f9bd5ca044d5fc19e69333f5d427e1fdb90b0864b" - } - ], - "fileName": "Modules/_decimal/libmpdec/constants.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "af9cbd016fb0ef0b30ced49c0aa4ce2ca3c20125" - }, - { - "algorithm": "SHA256", - "checksumValue": "19dc46df04abb7ee08e9a403f87c8aac8d4a077efcce314c597f8b73e22884f2" - } - ], - "fileName": "Modules/_decimal/libmpdec/constants.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-context.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "666162870230bebd3f2383020d908806fd03909e" - }, - { - "algorithm": "SHA256", - "checksumValue": "9a265d366f31894aad78bca7fcdc1457bc4a3aa3887ca231b7d78e41f79541c0" - } - ], - "fileName": "Modules/_decimal/libmpdec/context.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "0545547a8b37b922fbe2574fbad8fc3bf16f1d33" - }, - { - "algorithm": "SHA256", - "checksumValue": "66fe27b9bb37039cad5be32b105ed509e5aefa15c1957a9058af8ee23cddc97a" - } - ], - "fileName": "Modules/_decimal/libmpdec/convolute.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "05ff0936c5bb08f40d460f5843004a1cc0751d9b" - }, - { - "algorithm": "SHA256", - "checksumValue": "c00d17450c2b8e1d7f1eb8a084f7e6a68f257a453f8701600e860bf357c531d7" - } - ], - "fileName": "Modules/_decimal/libmpdec/convolute.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "fe8176849bc99a306332ba25caa4e91bfa3c6f7d" - }, - { - "algorithm": "SHA256", - "checksumValue": "1f4e65c44864c3e911a6e91f33adec76765293e90553459e3ebce35a58898dba" - } - ], - "fileName": "Modules/_decimal/libmpdec/crt.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "1930b9e0910014b3479aec4e940f02118d9e4a08" - }, - { - "algorithm": "SHA256", - "checksumValue": "7d31f1d0dd73b62964dab0f7a1724473bf87f1f95d8febf0b40c15430ae9a47c" - } - ], - "fileName": "Modules/_decimal/libmpdec/crt.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "415c51e7d7f517b6366bec2a809610d0d38ada14" - }, - { - "algorithm": "SHA256", - "checksumValue": "0a9fef8a374f55277e9f6000b7277bb037b9763c32b156c29950422b057498bd" - } - ], - "fileName": "Modules/_decimal/libmpdec/difradix2.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "d8a998c3bee4c3d9059ba7bf9ae6a8b64649c2ba" - }, - { - "algorithm": "SHA256", - "checksumValue": "5c6766496224de657400995b58b64db3e7084004bf00daebdd7e08d0c5995243" - } - ], - "fileName": "Modules/_decimal/libmpdec/difradix2.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-README.txt", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "158f6ad18edf348efa4fdd7cf61114c77c1d22e9" - }, - { - "algorithm": "SHA256", - "checksumValue": "7b0da2758097a2688f06b3c7ca46b2ebc8329addbd28bb4f5fe95626cc81f8a9" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/README.txt" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-compare.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "ef80ba26847287fb351ab0df0a78b5f08ba0b5b7" - }, - { - "algorithm": "SHA256", - "checksumValue": "452666ee4eb10a8cf0a926cb3bcf5e95b5c361fa129dbdfe27b654e6d640417e" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/compare.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-div.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "6ca3a369b3d1e140fdc93c4fdbedb724f7daf969" - }, - { - "algorithm": "SHA256", - "checksumValue": "6d369f5a24d0bb1e7cb6a4f8b0e97a273260e7668c8a540a8fcc92e039f7af2e" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/div.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-divmod.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "3872a28b4f77e07e1760256067ea338a8dd183f8" - }, - { - "algorithm": "SHA256", - "checksumValue": "5db54bae75ac3d7fa12f1bb0f7ce1bf797df86a81030e8c3ce44d3b1f9b958b7" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/divmod.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-multiply.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "25dbc94fd4ee5dec21061d2d40dd5d0f88849cb1" - }, - { - "algorithm": "SHA256", - "checksumValue": "22ed39b18fa740a27aacfd29a7bb40066be24500ba49b9b1f24e2af1e039fcd9" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/multiply.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-pow.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "13d3b7657dc2dc5000fea428f57963d520792ef7" - }, - { - "algorithm": "SHA256", - "checksumValue": "cd8c037649b3d4d6897c9acd2f92f3f9d5390433061d5e48623a5d526a3f4f9c" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/pow.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-powmod.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "1f7e6c3d3e38df52bbcec0f5a180a8f328679618" - }, - { - "algorithm": "SHA256", - "checksumValue": "e29614b43abf1856b656a84d6b67c22cc5dc7af8cbae8ddc7acf17022220ee12" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/powmod.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-shift.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "0bd9ce89c7987d1109eb7b0c8f1f9a1298e1422e" - }, - { - "algorithm": "SHA256", - "checksumValue": "203f2dbf11d115580cb3c7c524ac6ccca2a7b31d89545db1b6263381b5de2b6a" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/shift.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-sqrt.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "b401ba0814e17c9164c0df26e01cc0a355382f46" - }, - { - "algorithm": "SHA256", - "checksumValue": "f3dc2ce321833bbd4b3d1d9ea6fa2e0bcc1bfe1e39abb8d55be53e46c33949db" - } - ], - "fileName": "Modules/_decimal/libmpdec/examples/sqrt.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "060615ddef089a5a8f879a57e4968d920972a0e2" - }, - { - "algorithm": "SHA256", - "checksumValue": "a9f923524d53a9445769f27405375ec3d95fa804bb11db5ee249ae047f11cfce" - } - ], - "fileName": "Modules/_decimal/libmpdec/fnt.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "b205043ebeaf065b16505a299342a992654f19b0" - }, - { - "algorithm": "SHA256", - "checksumValue": "3b03e69adf78fde68c8f87d33595d557237581d33fc067e1039eed9e9f2cc44c" - } - ], - "fileName": "Modules/_decimal/libmpdec/fnt.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "702c27599b43280c94906235d7e1a74193ba701b" - }, - { - "algorithm": "SHA256", - "checksumValue": "cf2e69b946ec14b087e523c0ff606553070d13c23e851fb0ba1df51a728017e6" - } - ], - "fileName": "Modules/_decimal/libmpdec/fourstep.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "ee5291c265ef1f5ae373bc243a4d96975eb3e7b5" - }, - { - "algorithm": "SHA256", - "checksumValue": "dbaced03b52d0f880c377b86c943bcb36f24d557c99a5e9732df3ad5debb5917" - } - ], - "fileName": "Modules/_decimal/libmpdec/fourstep.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-io.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "5d6fdd98730584f74f7b731da6e488fe234504b3" - }, - { - "algorithm": "SHA256", - "checksumValue": "d74f365463166891f62e1326d22b2d39d865776b7ea5e0df2aea5eede4d85b0f" - } - ], - "fileName": "Modules/_decimal/libmpdec/io.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-io.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "28c653cd40b1ce46575e41f5dbfda5f6dd0db4d1" - }, - { - "algorithm": "SHA256", - "checksumValue": "259eab89fe27914e0e39e61199094a357ac60d86b2aab613c909040ff64a4a0c" - } - ], - "fileName": "Modules/_decimal/libmpdec/io.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-REFERENCES.txt", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "218d1d7bedb335cd2c31eae89a15873c3139e13f" - }, - { - "algorithm": "SHA256", - "checksumValue": "a57e8bed93ded481ef264166aec2c49d1a7f3252f29a873ee41fff053cfd9c20" - } - ], - "fileName": "Modules/_decimal/libmpdec/literature/REFERENCES.txt" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-bignum.txt", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "f67eab2431336cf6eeafb30cdafd7e54c251def3" - }, - { - "algorithm": "SHA256", - "checksumValue": "dc34aa122c208ce79e3fc6baee8628094ffaf6a662862dd5647836241f6ebd79" - } - ], - "fileName": "Modules/_decimal/libmpdec/literature/bignum.txt" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-fnt.py", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "a58cfbcd8ea57d66ddfd11fb5a170138c8bbfb3a" - }, - { - "algorithm": "SHA256", - "checksumValue": "122de20eebf87274af2d02072251a94500e7df2d5ef29e81aeabeda991c079e3" - } - ], - "fileName": "Modules/_decimal/libmpdec/literature/fnt.py" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-matrix-transform.txt", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "9a947f6b660150cbd457c4458da2956a36c5824d" - }, - { - "algorithm": "SHA256", - "checksumValue": "592659e7192e3a939b797f5bc7455455834a285f5d8b643ccd780b5114914f73" - } - ], - "fileName": "Modules/_decimal/libmpdec/literature/matrix-transform.txt" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-64.txt", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "69fe9afb8353b5a2b57917469c51c64ac518169d" - }, - { - "algorithm": "SHA256", - "checksumValue": "229a80ca940c594a32e3345412370cbc097043fe59c66a6153cbcf01e7837266" - } - ], - "fileName": "Modules/_decimal/libmpdec/literature/mulmod-64.txt" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-ppro.txt", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "720d468a1f51098036c7a0c869810fff22ed9b79" - }, - { - "algorithm": "SHA256", - "checksumValue": "f3549fc73f697a087267c7b042e30a409e191cbba69a2c0902685e507fbae9f7" - } - ], - "fileName": "Modules/_decimal/libmpdec/literature/mulmod-ppro.txt" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-six-step.txt", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "6815ec3a39baebebe7b3f51d45d10c180a659f17" - }, - { - "algorithm": "SHA256", - "checksumValue": "bf15f73910a173c98fca9db56122b6cc71983668fa8b934c46ca21a57398ec54" - } - ], - "fileName": "Modules/_decimal/libmpdec/literature/six-step.txt" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-umodarith.lisp", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "c91ac4438e661ce78f86e981257546e5adff39ae" - }, - { - "algorithm": "SHA256", - "checksumValue": "783a1b4b9b7143677b0c3d30ffaf28aa0cb01956409031fa38ed8011970bdee0" - } - ], - "fileName": "Modules/_decimal/libmpdec/literature/umodarith.lisp" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "7e8dfb4b7a801b48c501969b001153203b14679e" - }, - { - "algorithm": "SHA256", - "checksumValue": "5ba2f4c80302e71fb216aa247c858e0bf6c8cfabffe7980ac17d4d023c0fef2b" - } - ], - "fileName": "Modules/_decimal/libmpdec/mpalloc.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "bccb6a6ae76fd7f6c8a9102a78958bcad7862950" - }, - { - "algorithm": "SHA256", - "checksumValue": "f7412521de38afb837fcabc2b1d48b971b86bfaa55f3f40d58ff9e46e92debd3" - } - ], - "fileName": "Modules/_decimal/libmpdec/mpalloc.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "f4539afb1ace58c52d18ffd0cc7704f53ca55182" - }, - { - "algorithm": "SHA256", - "checksumValue": "4f89b8095e408a18deff79cfb605299e615bae747898eb105d8936064f7fb626" - } - ], - "fileName": "Modules/_decimal/libmpdec/mpdecimal.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "4b80e25ac49b7e1ea0d1e84967ee32a3d111fc4c" - }, - { - "algorithm": "SHA256", - "checksumValue": "ea0b9c6b296c13aed6ecaa50b463e39a9c1bdc059b84f50507fd8247b2e660f9" - } - ], - "fileName": "Modules/_decimal/libmpdec/mpdecimal.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-mpsignal.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "5c7305a6db0fddf64c6d97e29d3b0c402e3d5d6e" - }, - { - "algorithm": "SHA256", - "checksumValue": "653171cf2549719478417db7e9800fa0f9d99c02dec6da6876329ccf2c07b93f" - } - ], - "fileName": "Modules/_decimal/libmpdec/mpsignal.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "d736b874c43777ca021dde5289ea718893f39219" - }, - { - "algorithm": "SHA256", - "checksumValue": "bdbf2e246f341a3ba3f6f9d8759e7cb222eb9b15f9ed1e7c9f6a59cbb9f8bc91" - } - ], - "fileName": "Modules/_decimal/libmpdec/numbertheory.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "d341508d8c6dd4c4cbd8b99afc8029945f9bbe0d" - }, - { - "algorithm": "SHA256", - "checksumValue": "2f7d5b40af508fa6ac86f5d62101fa3bf683c63b24aa87c9548e3fdd13abc57b" - } - ], - "fileName": "Modules/_decimal/libmpdec/numbertheory.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "cbd05d68bb3940d0d7d0818b14cc03b090a4dd74" - }, - { - "algorithm": "SHA256", - "checksumValue": "7602aaf98ec9525bc4b3cab9631615e1be2efd9af894002ef4e3f5ec63924fcf" - } - ], - "fileName": "Modules/_decimal/libmpdec/sixstep.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "4c059463ec4b4522562dab24760fc64c172c9eee" - }, - { - "algorithm": "SHA256", - "checksumValue": "a191366348b3d3dd49b9090ec5c77dbd77bb3a523c01ff32adafa137e5097ce7" - } - ], - "fileName": "Modules/_decimal/libmpdec/sixstep.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.c", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "cc5593ac9fdb60480cc23fc9d6f27d85670bd35f" - }, - { - "algorithm": "SHA256", - "checksumValue": "2d12fcae512143a9376c8a0d4c1ba3008e420e024497a7e7ec64c6bec23fcddc" - } - ], - "fileName": "Modules/_decimal/libmpdec/transpose.c" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "2f616425756b6cbdf7d189744870b98b613455bd" - }, - { - "algorithm": "SHA256", - "checksumValue": "fafeb2b901b2b41bf0df00be7d99b84df1a78e3cc1e582e09cbfc3b6d44d4abe" - } - ], - "fileName": "Modules/_decimal/libmpdec/transpose.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-typearith.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "b1e9341e173cc8e219ad4aa45fad36d92cce10d3" - }, - { - "algorithm": "SHA256", - "checksumValue": "25e0a0703b51744277834e6b2398d7b7d2c17f92bf30f8b6f949e0486ae2b346" - } - ], - "fileName": "Modules/_decimal/libmpdec/typearith.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-umodarith.h", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "46f6483fce136cd3cc2f7516ee119a487d86333e" - }, - { - "algorithm": "SHA256", - "checksumValue": "bfe1ddb2ca92906456b80745adcbe02c83cadac3ef69caa21bc09b7292cc152b" - } - ], - "fileName": "Modules/_decimal/libmpdec/umodarith.h" - }, - { - "SPDXID": "SPDXRef-FILE-Modules-decimal-libmpdec-vcdiv64.asm", - "checksums": [ - { - "algorithm": "SHA1", - "checksumValue": "d0cc1052fcba08b773d935b0ae2dc6b80d0f2f68" - }, - { - "algorithm": "SHA256", - "checksumValue": "aacc3e47ea8f41e8840c6c67f64ec96d54696a16889903098fa1aab56949a00f" - } - ], - "fileName": "Modules/_decimal/libmpdec/vcdiv64.asm" } ], "packages": [ @@ -1730,14 +1002,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" + "checksumValue": "a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_0/expat-2.8.0.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_1/expat-2.8.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,21 +1017,21 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.8.0" + "versionInfo": "2.8.1" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "61e48893f37cb2280d106cefacf6fb5afe84edf625fec39572d0ee94e1018f26" + "checksumValue": "d6db56a5d061dcc0890eabdbb5f58a9fa6c606d9f2fbbe9d626925b870ffadfb" } ], - "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/8ba599b2f6c9701b3dc961db895b0856a2210f76.zip", + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/504c2987452f87fe44bce9b9f12e19d6e051761f.zip", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:8ba599b2f6c9701b3dc961db895b0856a2210f76:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:504c2987452f87fe44bce9b9f12e19d6e051761f:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1767,7 +1039,7 @@ "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", - "versionInfo": "8ba599b2f6c9701b3dc961db895b0856a2210f76" + "versionInfo": "504c2987452f87fe44bce9b9f12e19d6e051761f" }, { "SPDXID": "SPDXRef-PACKAGE-macholib", @@ -1790,28 +1062,6 @@ "originator": "Person: Ronald Oussoren (ronaldoussoren@mac.com)", "primaryPackagePurpose": "SOURCE", "versionInfo": "1.0" - }, - { - "SPDXID": "SPDXRef-PACKAGE-mpdecimal", - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "9f9cd4c041f99b5c49ffb7b59d9f12d95b683d88585608aa56a6307667b2b21f" - } - ], - "downloadLocation": "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.1.tar.gz", - "externalRefs": [ - { - "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:2.5.1:*:*:*:*:*:*:*", - "referenceType": "cpe23Type" - } - ], - "licenseConcluded": "NOASSERTION", - "name": "mpdecimal", - "originator": "Organization: bytereef.org", - "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.5.1" } ], "relationships": [ @@ -2169,266 +1419,6 @@ "relatedSpdxElement": "SPDXRef-FILE-Lib-ctypes-macholib-framework.py", "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-macholib" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-README.txt", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-basearith.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bench.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bench-full.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-bits.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-constants.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-context.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-convolute.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-crt.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-difradix2.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-README.txt", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-compare.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-div.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-divmod.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-multiply.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-pow.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-powmod.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-shift.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-examples-sqrt.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fnt.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-fourstep.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-io.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-io.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-REFERENCES.txt", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-bignum.txt", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-fnt.py", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-matrix-transform.txt", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-64.txt", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-mulmod-ppro.txt", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-six-step.txt", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-literature-umodarith.lisp", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpalloc.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpdecimal.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-mpsignal.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-numbertheory.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-sixstep.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.c", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-transpose.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-typearith.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-umodarith.h", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" - }, - { - "relatedSpdxElement": "SPDXRef-FILE-Modules-decimal-libmpdec-vcdiv64.asm", - "relationshipType": "CONTAINS", - "spdxElementId": "SPDXRef-PACKAGE-mpdecimal" } ], "spdxVersion": "SPDX-2.3" diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index e6c63227d20446a..d59a7c788fa9e02 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -296,10 +296,12 @@ [const.Py_sq_inplace_repeat] added = '3.2' [const.Py_mp_length] + # Note: value changed in 3.15 (old value is still accepted) added = '3.2' [const.Py_mp_subscript] added = '3.2' [const.Py_mp_ass_subscript] + # Note: value changed in 3.15 (old value is still accepted) added = '3.2' [typedef.Py_uintptr_t] @@ -1811,7 +1813,6 @@ [const.METH_COEXIST] added = '3.2' # METH_STACKLESS is undocumented -# METH_FASTCALL is not part of limited API. # The following are defined in private headers, but historically # they were exported as part of the stable ABI. @@ -1914,8 +1915,10 @@ [data.PyModuleDef_Type] added = '3.5' [const.Py_mod_create] + # Note: value changed in 3.15 (old value is still accepted) added = '3.5' [const.Py_mod_exec] + # Note: value changed in 3.15 (old value is still accepted) added = '3.5' [struct.PyModuleDef_Slot] added = '3.5' @@ -2145,8 +2148,6 @@ # New method flags in 3.7 (PEP 590): -[const.METH_FASTCALL] - added = '3.7' [const.METH_METHOD] added = '3.7' @@ -2296,6 +2297,10 @@ [data.PyStructSequence_UnnamedField] added = '3.11' +# Added in 3.7 but in the Stable ABI from 3.10 +[const.METH_FASTCALL] + added = '3.10' + # Add stable Py_buffer API in Python 3.11 (https://bugs.python.org/issue45459) [struct.Py_buffer] added = '3.11' @@ -2325,8 +2330,10 @@ [function.PyMemoryView_FromBuffer] added = '3.11' [const.Py_bf_getbuffer] + # Note: value changed in 3.15 (old value is still accepted) added = '3.11' [const.Py_bf_releasebuffer] + # Note: value changed in 3.15 (old value is still accepted) added = '3.11' # Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) @@ -2463,6 +2470,7 @@ [const.Py_TPFLAGS_ITEMS_AT_END] added = '3.12' [const.Py_mod_multiple_interpreters] + # Note: value changed in 3.15 (old value is still accepted) added = '3.12' [function.PyImport_AddModuleRef] @@ -2544,6 +2552,7 @@ [function.PyEval_GetFrameLocals] added = '3.13' [const.Py_mod_gil] + # Note: value changed in 3.15 (old value is still accepted) added = '3.13' [function.Py_TYPE] @@ -2689,8 +2698,84 @@ added = '3.15' [function.PyType_GetModuleByToken_DuringGC] added = '3.15' +[function.PyCriticalSection_Begin] + added = '3.15' +[function.PyCriticalSection_End] + added = '3.15' +[struct.PyCriticalSection] + added = '3.15' + struct_abi_kind = 'full-abi' +[macro.Py_BEGIN_CRITICAL_SECTION] + added = '3.15' +[macro.Py_END_CRITICAL_SECTION] + added = '3.15' +[function.PyCriticalSection2_Begin] + added = '3.15' +[function.PyCriticalSection2_End] + added = '3.15' +[struct.PyCriticalSection2] + added = '3.15' + struct_abi_kind = 'full-abi' +[macro.Py_BEGIN_CRITICAL_SECTION2] + added = '3.15' +[macro.Py_END_CRITICAL_SECTION2] + added = '3.15' +[struct.PySlot] + added = '3.15' + struct_abi_kind = 'full-abi' +[function.PyType_FromSlots] + added = '3.15' +[const.PySlot_OPTIONAL] + added = '3.15' +[const.PySlot_STATIC] + added = '3.15' +[const.PySlot_INTPTR] + added = '3.15' +[macro.PySlot_DATA] + added = '3.15' +[macro.PySlot_FUNC] + added = '3.15' +[macro.PySlot_SIZE] + added = '3.15' +[macro.PySlot_INT64] + added = '3.15' +[macro.PySlot_UINT64] + added = '3.15' +[macro.PySlot_STATIC_DATA] + added = '3.15' +[macro.PySlot_END] + added = '3.15' +[macro.PySlot_PTR] + added = '3.15' +[macro.PySlot_PTR_STATIC] + added = '3.15' +[const.Py_slot_end] + added = '3.15' +[const.Py_slot_invalid] + added = '3.15' +[const.Py_slot_subslots] + added = '3.15' +[const.Py_tp_slots] + added = '3.15' +[const.Py_mod_slots] + added = '3.15' +[const.Py_tp_name] + added = '3.15' +[const.Py_tp_basicsize] + added = '3.15' +[const.Py_tp_extra_basicsize] + added = '3.15' +[const.Py_tp_itemsize] + added = '3.15' +[const.Py_tp_flags] + added = '3.15' +[const.Py_tp_metaclass] + added = '3.15' +[const.Py_tp_module] + added = '3.15' # PEP 757 import/export API. + [function.PyLong_GetNativeLayout] added = '3.15' [function.PyLong_Export] @@ -2715,5 +2800,35 @@ # (The definition of 'full-abi' was clarified when this entry was added.) struct_abi_kind = 'full-abi' +# PEP 788 finalization protection + +[struct.PyInterpreterGuard] + added = '3.15' + struct_abi_kind = 'opaque' +[function.PyInterpreterGuard_FromCurrent] + added = '3.15' +[function.PyInterpreterGuard_FromView] + added = '3.15' +[function.PyInterpreterGuard_Close] + added = '3.15' +[struct.PyInterpreterView] + added = '3.15' + struct_abi_kind = 'opaque' +[function.PyInterpreterView_FromCurrent] + added = '3.15' +[function.PyInterpreterView_FromMain] + added = '3.15' +[function.PyInterpreterView_Close] + added = '3.15' +[function.PyThreadState_Ensure] + added = '3.15' +[function.PyThreadState_EnsureFromView] + added = '3.15' +[function.PyThreadState_Release] + added = '3.15' +[struct.PyThreadStateToken] + added = '3.15' + struct_abi_kind = 'opaque' + [function.PyObject_CallFinalizerFromDealloc] added = '3.15' diff --git a/Misc/valgrind-python.supp b/Misc/valgrind-python.supp index 8b2027cd4527679..4df8a118f089b70 100644 --- a/Misc/valgrind-python.supp +++ b/Misc/valgrind-python.supp @@ -317,13 +317,8 @@ { Avoid problems w/readline doing a putenv and leaking on exit Memcheck:Leak - fun:malloc - fun:xmalloc - fun:sh_set_lines_and_columns - fun:_rl_get_screen_size - fun:_rl_init_terminal_io - obj:/lib/libreadline.so.4.3 - fun:rl_initialize + ... + fun:setup_readline } # Valgrind emits "Conditional jump or move depends on uninitialised value(s)" diff --git a/Modules/Setup b/Modules/Setup index 33737c21cb4066e..e97a78e628693dc 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -273,6 +273,7 @@ PYTHONPATH=$(COREPYTHONPATH) #xx xxmodule.c #xxlimited xxlimited.c #xxlimited_35 xxlimited_35.c +#xxlimited_3_13 xxlimited_3_13.c #xxsubtype xxsubtype.c # Testing diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index a274c312d99313e..8efea27824f0e88 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -56,9 +56,7 @@ @MODULE_CMATH_TRUE@cmath cmathmodule.c @MODULE__STATISTICS_TRUE@_statistics _statisticsmodule.c -# _decimal uses libmpdec -# either static libmpdec.a from Modules/_decimal/libmpdec or libmpdec.so -# with ./configure --with-system-libmpdec +# _decimal uses the system's libmpdec.so @MODULE__DECIMAL_TRUE@_decimal _decimal/_decimal.c # compression libs and binascii (optional CRC32 from zlib) @@ -175,8 +173,8 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c @@ -190,6 +188,7 @@ # Limited API template modules; must be built as shared modules. @MODULE_XXLIMITED_TRUE@xxlimited xxlimited.c @MODULE_XXLIMITED_35_TRUE@xxlimited_35 xxlimited_35.c +@MODULE_XXLIMITED_3_13_TRUE@xxlimited_3_13 xxlimited_3_13.c # for performance diff --git a/Modules/_abc.c b/Modules/_abc.c index 3c4e0280525e1eb..5826efbfecb6901 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -915,14 +915,14 @@ _abc.get_cache_token Returns the current ABC cache token. -The token is an opaque object (supporting equality testing) identifying the -current version of the ABC cache for virtual subclasses. The token changes -with every call to register() on any ABC. +The token is an opaque object (supporting equality testing) identifying +the current version of the ABC cache for virtual subclasses. The token +changes with every call to register() on any ABC. [clinic start generated code]*/ static PyObject * _abc_get_cache_token_impl(PyObject *module) -/*[clinic end generated code: output=c7d87841e033dacc input=70413d1c423ad9f9]*/ +/*[clinic end generated code: output=c7d87841e033dacc input=d87acc04492f6bf3]*/ { _abcmodule_state *state = get_abc_state(module); return PyLong_FromUnsignedLongLong(get_invalidation_counter(state)); diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 9679a7dde31b0d0..57c8d4a41a3a0df 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -955,12 +955,13 @@ Return the result this future represents. If the future has been cancelled, raises CancelledError. If the future's result isn't yet available, raises InvalidStateError. If -the future is done and has an exception set, this exception is raised. +the future is done and has an exception set, this exception is +raised. [clinic start generated code]*/ static PyObject * _asyncio_Future_result_impl(FutureObj *self) -/*[clinic end generated code: output=f35f940936a4b1e5 input=61d89f48e4c8b670]*/ +/*[clinic end generated code: output=f35f940936a4b1e5 input=ee20e126776cbb04]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *result; @@ -1095,15 +1096,15 @@ _asyncio.Future.add_done_callback Add a callback to be run when the future becomes done. -The callback is called with a single argument - the future object. If -the future is already done when this is called, the callback is +The callback is called with a single argument - the future object. +If the future is already done when this is called, the callback is scheduled with call_soon. [clinic start generated code]*/ static PyObject * _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn, PyObject *context) -/*[clinic end generated code: output=922e9a4cbd601167 input=37d97f941beb7b3e]*/ +/*[clinic end generated code: output=922e9a4cbd601167 input=f4f6adb074cd3e0f]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); if (context == NULL) { @@ -1252,15 +1253,15 @@ _asyncio.Future.cancel Cancel the future and schedule callbacks. -If the future is already done or cancelled, return False. Otherwise, -change the future's state to cancelled, schedule the callbacks and -return True. +If the future is already done or cancelled, return False. +Otherwise, change the future's state to cancelled, schedule the +callbacks and return True. [clinic start generated code]*/ static PyObject * _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, PyObject *msg) -/*[clinic end generated code: output=074956f35904b034 input=44ab4003da839970]*/ +/*[clinic end generated code: output=074956f35904b034 input=0c9157547a964c4c]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -1292,13 +1293,13 @@ _asyncio.Future.done Return True if the future is done. -Done means either that a result / exception are available, or that the -future was cancelled. +Done means either that a result / exception are available, or that +the future was cancelled. [clinic start generated code]*/ static PyObject * _asyncio_Future_done_impl(FutureObj *self) -/*[clinic end generated code: output=244c5ac351145096 input=7204d3cc63bef7f3]*/ +/*[clinic end generated code: output=244c5ac351145096 input=acf2c2347f3c01d8]*/ { if (!future_is_alive(self) || self->fut_state == STATE_PENDING) { Py_RETURN_FALSE; @@ -1741,7 +1742,8 @@ static PyMethodDef FutureType_methods[] = { _ASYNCIO_FUTURE_DONE_METHODDEF _ASYNCIO_FUTURE_GET_LOOP_METHODDEF _ASYNCIO_FUTURE__MAKE_CANCELLED_ERROR_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Futures are generic over the type of their results")}, {NULL, NULL} /* Sentinel */ }; @@ -2364,6 +2366,11 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, return -1; } _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); +#ifdef Py_GIL_DISABLED + // This is required so that _Py_TryIncref(self) + // works correctly in non-owning threads. + _PyObject_SetMaybeWeakref((PyObject *)self); +#endif if (eager_start) { PyObject *res = PyObject_CallMethodNoArgs(loop, &_Py_ID(is_running)); if (res == NULL) { @@ -2382,11 +2389,6 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (task_call_step_soon(state, self, NULL)) { return -1; } -#ifdef Py_GIL_DISABLED - // This is required so that _Py_TryIncref(self) - // works correctly in non-owning threads. - _PyObject_SetMaybeWeakref((PyObject *)self); -#endif register_task(ts, self); return 0; } @@ -2926,7 +2928,8 @@ static PyMethodDef TaskType_methods[] = { _ASYNCIO_TASK_SET_NAME_METHODDEF _ASYNCIO_TASK_GET_CORO_METHODDEF _ASYNCIO_TASK_GET_CONTEXT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Tasks are generic over the return type of their wrapped coroutines")}, {NULL, NULL} /* Sentinel */ }; @@ -3844,6 +3847,7 @@ _asyncio__leave_task_impl(PyObject *module, PyObject *loop, PyObject *task) /*[clinic input] +@permit_long_summary _asyncio._swap_current_task loop: object @@ -3858,7 +3862,7 @@ This is intended for use during eager coroutine execution. static PyObject * _asyncio__swap_current_task_impl(PyObject *module, PyObject *loop, PyObject *task) -/*[clinic end generated code: output=9f88de958df74c7e input=c9c72208d3d38b6c]*/ +/*[clinic end generated code: output=9f88de958df74c7e input=ec14ed25855e3068]*/ { _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); return swap_current_task(ts, loop, task); diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 329aa8e117ec3cf..a953f8bfa11aeaa 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -157,8 +157,8 @@ _bisect.bisect_right -> Py_ssize_t Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e <= x, and all e in -a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will -insert just after the rightmost x already there. +a[i:] have e > x. So if x already appears in the list, a.insert(i, x) +will insert just after the rightmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. @@ -169,7 +169,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_right_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=3a4bc09cc7c8a73d input=b476bc45667273ac]*/ +/*[clinic end generated code: output=3a4bc09cc7c8a73d input=27717afe1a61bfaa]*/ { return internal_bisect_right(a, x, lo, hi, key); } @@ -338,8 +338,8 @@ _bisect.bisect_left -> Py_ssize_t Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e < x, and all e in -a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will -insert just before the leftmost x already there. +a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) +will insert just before the leftmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. @@ -350,7 +350,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_left_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=70749d6e5cae9284 input=9b4d49b5ddecfad7]*/ +/*[clinic end generated code: output=70749d6e5cae9284 input=259fedbe35e882e1]*/ { return internal_bisect_left(a, x, lo, hi, key); } diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 4bff90e6fd2b2e0..9db3ac39da52099 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -108,6 +108,7 @@ typedef struct { typedef struct { PyObject_HEAD bz_stream bzs; + int bzerror; char eof; /* Py_T_BOOL expects a char */ PyObject *unused_data; char needs_input; @@ -435,8 +436,11 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length) d->bzs_avail_in_real += bzs->avail_in; - if (catch_bz2_error(bzret)) + if (catch_bz2_error(bzret)) { + d->bzerror = bzret; + FT_ATOMIC_STORE_CHAR_RELAXED(d->needs_input, 0); goto error; + } if (bzret == BZ_STREAM_END) { FT_ATOMIC_STORE_CHAR_RELAXED(d->eof, 1); break; @@ -577,7 +581,6 @@ decompress(BZ2Decompressor *d, char *data, size_t len, Py_ssize_t max_length) } /*[clinic input] -@permit_long_docstring_body _bz2.BZ2Decompressor.decompress data: Py_buffer @@ -585,32 +588,40 @@ _bz2.BZ2Decompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=23e41045deb240a3 input=3703e78f91757655]*/ +/*[clinic end generated code: output=23e41045deb240a3 input=7f68faa9ff7a1b51]*/ { PyObject *result = NULL; PyMutex_Lock(&self->mutex); - if (self->eof) + if (self->eof) { PyErr_SetString(PyExc_EOFError, "End of stream already reached"); - else + } + else if (self->bzerror) { + // Re-entering BZ2_bzDecompress() after an error can write out of bounds. + PyErr_SetString(PyExc_ValueError, + "Decompressor is unusable after a previous error"); + } + else { result = decompress(self, data->buf, data->len, max_length); + } PyMutex_Unlock(&self->mutex); return result; } @@ -638,6 +649,7 @@ _bz2_BZ2Decompressor_impl(PyTypeObject *type) } self->mutex = (PyMutex){0}; + self->bzerror = 0; self->needs_input = 1; self->bzs_avail_in_real = 0; self->input_buffer = NULL; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index ff52bfd8291ac14..272182f7bf49acc 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -55,14 +55,15 @@ _codecs.register Register a codec search function. -Search functions are expected to take one argument, the encoding name in -all lower case letters, and either return None, or a tuple of functions -(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object). +Search functions are expected to take one argument, the encoding +name in all lower case letters, and either return None, or a tuple +of functions (encoder, decoder, stream_reader, stream_writer) (or +a CodecInfo object). [clinic start generated code]*/ static PyObject * _codecs_register(PyObject *module, PyObject *search_function) -/*[clinic end generated code: output=d1bf21e99db7d6d3 input=369578467955cae4]*/ +/*[clinic end generated code: output=d1bf21e99db7d6d3 input=2321d8c8c0420dfc]*/ { if (PyCodec_Register(search_function)) return NULL; @@ -116,16 +117,16 @@ _codecs.encode Encodes obj using the codec registered for encoding. The default encoding is 'utf-8'. errors may be given to set a -different error handling scheme. Default is 'strict' meaning that encoding -errors raise a ValueError. Other possible values are 'ignore', 'replace' -and 'backslashreplace' as well as any other name registered with -codecs.register_error that can handle ValueErrors. +different error handling scheme. Default is 'strict' meaning that +encoding errors raise a ValueError. Other possible values are 'ignore', +'replace' and 'backslashreplace' as well as any other name registered +with codecs.register_error that can handle ValueErrors. [clinic start generated code]*/ static PyObject * _codecs_encode_impl(PyObject *module, PyObject *obj, const char *encoding, const char *errors) -/*[clinic end generated code: output=385148eb9a067c86 input=cd5b685040ff61f0]*/ +/*[clinic end generated code: output=385148eb9a067c86 input=e5271d443e391d7f]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -143,16 +144,16 @@ _codecs.decode Decodes obj using the codec registered for encoding. Default encoding is 'utf-8'. errors may be given to set a -different error handling scheme. Default is 'strict' meaning that encoding -errors raise a ValueError. Other possible values are 'ignore', 'replace' -and 'backslashreplace' as well as any other name registered with -codecs.register_error that can handle ValueErrors. +different error handling scheme. Default is 'strict' meaning that +encoding errors raise a ValueError. Other possible values are 'ignore', +'replace' and 'backslashreplace' as well as any other name registered +with codecs.register_error that can handle ValueErrors. [clinic start generated code]*/ static PyObject * _codecs_decode_impl(PyObject *module, PyObject *obj, const char *encoding, const char *errors) -/*[clinic end generated code: output=679882417dc3a0bd input=7702c0cc2fa1add6]*/ +/*[clinic end generated code: output=679882417dc3a0bd input=3e6254628f9ca538]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -962,14 +963,15 @@ _codecs.register_error Register the specified error handler under the name errors. handler must be a callable object, that will be called with an exception -instance containing information about the location of the encoding/decoding -error and must return a (replacement, new position) tuple. +instance containing information about the location of the +encoding/decoding error and must return a (replacement, new position) +tuple. [clinic start generated code]*/ static PyObject * _codecs_register_error_impl(PyObject *module, const char *errors, PyObject *handler) -/*[clinic end generated code: output=fa2f7d1879b3067d input=5e6709203c2e33fe]*/ +/*[clinic end generated code: output=fa2f7d1879b3067d input=5bea01dfe835d9d8]*/ { if (PyCodec_RegisterError(errors, handler)) return NULL; @@ -1007,13 +1009,13 @@ _codecs.lookup_error lookup_error(errors) -> handler -Return the error handler for the specified error handling name or raise a -LookupError, if no handler exists under this name. +Return the error handler for the specified error handling name or raise +a LookupError, if no handler exists under this name. [clinic start generated code]*/ static PyObject * _codecs_lookup_error_impl(PyObject *module, const char *name) -/*[clinic end generated code: output=087f05dc0c9a98cc input=4775dd65e6235aba]*/ +/*[clinic end generated code: output=087f05dc0c9a98cc input=86cfb6a7a9c67113]*/ { return PyCodec_LookupError(name); } diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 4ff05727ebc8ce3..e96a546a818d3d7 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -1077,6 +1077,7 @@ _deque_rotate(dequeobject *deque, Py_ssize_t n) } /*[clinic input] +@permit_long_summary @critical_section _collections.deque.rotate as deque_rotate @@ -1089,7 +1090,7 @@ Rotate the deque n steps to the right. If n is negative, rotates left. static PyObject * deque_rotate_impl(dequeobject *deque, Py_ssize_t n) -/*[clinic end generated code: output=96c2402a371eb15d input=5bf834296246e002]*/ +/*[clinic end generated code: output=96c2402a371eb15d input=3543c3b2297de8f1]*/ { if (!_deque_rotate(deque, n)) Py_RETURN_NONE; @@ -1250,7 +1251,7 @@ _collections.deque.index as deque_index deque: dequeobject value as v: object start: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='0') = NULL - stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='Py_SIZE(deque)') = NULL + stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='PY_SSIZE_T_MAX') = NULL / Return first index of value. @@ -1261,7 +1262,7 @@ Raises ValueError if the value is not present. static PyObject * deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, Py_ssize_t stop) -/*[clinic end generated code: output=df45132753175ef9 input=90f48833a91e1743]*/ +/*[clinic end generated code: output=df45132753175ef9 input=1c3b19632cf3484f]*/ { Py_ssize_t i, n; PyObject *item; @@ -1269,22 +1270,23 @@ deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; int cmp; + Py_ssize_t size = Py_SIZE(deque); if (start < 0) { - start += Py_SIZE(deque); + start += size; if (start < 0) start = 0; } if (stop < 0) { - stop += Py_SIZE(deque); + stop += size; if (stop < 0) stop = 0; } - if (stop > Py_SIZE(deque)) - stop = Py_SIZE(deque); + if (stop > size) + stop = size; if (start > stop) start = stop; - assert(0 <= start && start <= stop && stop <= Py_SIZE(deque)); + assert(0 <= start && start <= stop && stop <= size); for (i=0 ; i < start - BLOCKLEN ; i += BLOCKLEN) { b = b->rightlink; @@ -1854,7 +1856,7 @@ static PyMethodDef deque_methods[] = { DEQUE_ROTATE_METHODDEF DEQUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("deques are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; @@ -2330,6 +2332,12 @@ defdict_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) return result; } + +PyDoc_STRVAR(defdict_class_getitem_doc, +"defaultdicts are generic over two types, signifying (respectively) the types \ +of the dictionary's keys and values"); + + static PyMethodDef defdict_methods[] = { {"__missing__", defdict_missing, METH_O, defdict_missing_doc}, @@ -2340,7 +2348,7 @@ static PyMethodDef defdict_methods[] = { {"__reduce__", defdict_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + defdict_class_getitem_doc}, {NULL} }; @@ -2420,7 +2428,7 @@ defdict_or(PyObject* left, PyObject* right) self = right; other = left; } - if (!PyDict_Check(other)) { + if (!PyAnyDict_Check(other)) { Py_RETURN_NOTIMPLEMENTED; } // Like copy(), this calls the object's class. @@ -2582,7 +2590,7 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, if (key == NULL) break; - hash = _PyObject_HashFast(key); + hash = _PyObject_HashDictKey(key); if (hash == -1) { goto done; } diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 98ac821c525a647..e891249668c20f5 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -708,7 +708,7 @@ StructUnionType_paramfunc(ctypes_state *st, CDataObject *self) } assert(stginfo); /* Cannot be NULL for structure/union instances */ - parg->tag = 'V'; + parg->tag = "V"; parg->pffi_type = &stginfo->ffi_type_pointer; parg->value.p = ptr; parg->size = self->b_size; @@ -1282,7 +1282,7 @@ PyCPointerType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); parg->value.p = *(void **)self->b_ptr; @@ -1703,7 +1703,7 @@ PyCArrayType_paramfunc(ctypes_state *st, CDataObject *self) PyCArgObject *p = PyCArgObject_new(st); if (p == NULL) return NULL; - p->tag = 'P'; + p->tag = "P"; p->pffi_type = &ffi_type_pointer; p->value.p = (char *)self->b_ptr; p->obj = Py_NewRef(self); @@ -1909,7 +1909,7 @@ c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -1998,7 +1998,7 @@ c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'z'; + parg->tag = "z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2092,7 +2092,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'P'; + parg->tag = "P"; parg->obj = fd->setfunc(&parg->value, value, sizeof(void*)); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2110,7 +2110,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'z'; + parg->tag = "z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2127,7 +2127,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = fd->setfunc(&parg->value, value, 0); if (parg->obj == NULL) { Py_DECREF(parg); @@ -2152,7 +2152,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (PyCArg_CheckExact(st, value)) { /* byref(c_xxx()) */ PyCArgObject *a = (PyCArgObject *)value; - if (a->tag == 'P') { + if (strcmp(a->tag, "P") == 0) { return Py_NewRef(value); } } @@ -2165,7 +2165,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'P'; + parg->tag = "P"; Py_INCREF(value); // Function pointers don't change their contents, no need to lock parg->value.p = *(void **)func->b_ptr; @@ -2191,7 +2191,7 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) if (parg == NULL) return NULL; parg->pffi_type = &ffi_type_pointer; - parg->tag = 'Z'; + parg->tag = "Z"; parg->obj = Py_NewRef(value); /* Remember: b_ptr points to where the pointer is stored! */ Py_BEGIN_CRITICAL_SECTION(value); @@ -2332,7 +2332,8 @@ PyCSimpleType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = fmt[0]; + assert(strcmp(fd->code, fmt) == 0); + parg->tag = fd->code; parg->pffi_type = fd->pffi_type; parg->obj = Py_NewRef(self); memcpy(&parg->value, self->b_ptr, self->b_size); @@ -2578,7 +2579,8 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls, if (parg == NULL) return NULL; - parg->tag = fmt[0]; + assert(strcmp(fd->code, fmt) == 0); + parg->tag = fd->code; parg->pffi_type = fd->pffi_type; parg->obj = fd->setfunc(&parg->value, value, info->size); if (parg->obj) @@ -2832,7 +2834,7 @@ PyCFuncPtrType_paramfunc(ctypes_state *st, CDataObject *self) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(self); parg->value.p = *(void **)self->b_ptr; @@ -4303,7 +4305,7 @@ _byref(ctypes_state *st, PyObject *obj) return NULL; } - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = obj; parg->value.p = ((CDataObject *)obj)->b_ptr; @@ -5303,7 +5305,7 @@ Array_length(PyObject *myself) static PyMethodDef Array_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("Arrays are generic over the type of their elements")}, { NULL, NULL } }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a0c9d8b70fee469..991ff0d675c2f1c 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -446,7 +446,7 @@ EXPORT(char *)my_strtok(char *token, const char *delim) return strtok(token, delim); } -EXPORT(char *)my_strchr(const char *s, int c) +EXPORT(const char *) my_strchr(const char *s, int c) { return strchr(s, c); } diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index e208e27c5dbed42..ccc57e347b07acf 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -168,8 +168,9 @@ _ctypes_get_errobj(ctypes_state *st, int **pspace) } else { void *space = PyMem_Calloc(2, sizeof(int)); - if (space == NULL) - return NULL; + if (space == NULL) { + return PyErr_NoMemory(); + } errobj = PyCapsule_New(space, CTYPES_CAPSULE_NAME_PYMEM, pymem_destructor); if (errobj == NULL) { PyMem_Free(space); @@ -468,7 +469,7 @@ PyCArgObject_new(ctypes_state *st) if (p == NULL) return NULL; p->pffi_type = NULL; - p->tag = '\0'; + p->tag = ""; p->obj = NULL; memset(&p->value, 0, sizeof(p->value)); PyObject_GC_Track(p); @@ -512,45 +513,50 @@ static PyObject * PyCArg_repr(PyObject *op) { PyCArgObject *self = _PyCArgObject_CAST(op); - switch(self->tag) { + + if (strlen(self->tag) != 1) { + goto generic; + } + + switch(self->tag[0]) { case 'b': case 'B': - return PyUnicode_FromFormat("<cparam '%c' (%d)>", + return PyUnicode_FromFormat("<cparam '%s' (%d)>", self->tag, self->value.b); case 'h': case 'H': - return PyUnicode_FromFormat("<cparam '%c' (%d)>", + return PyUnicode_FromFormat("<cparam '%s' (%d)>", self->tag, self->value.h); case 'i': case 'I': - return PyUnicode_FromFormat("<cparam '%c' (%d)>", + return PyUnicode_FromFormat("<cparam '%s' (%d)>", self->tag, self->value.i); case 'l': case 'L': - return PyUnicode_FromFormat("<cparam '%c' (%ld)>", + return PyUnicode_FromFormat("<cparam '%s' (%ld)>", self->tag, self->value.l); case 'q': case 'Q': - return PyUnicode_FromFormat("<cparam '%c' (%lld)>", + return PyUnicode_FromFormat("<cparam '%s' (%lld)>", self->tag, self->value.q); case 'd': case 'f': { - PyObject *f = PyFloat_FromDouble((self->tag == 'f') ? self->value.f : self->value.d); + PyObject *f = PyFloat_FromDouble((strcmp(self->tag, "f") == 0) ? self->value.f : self->value.d); if (f == NULL) { return NULL; } - PyObject *result = PyUnicode_FromFormat("<cparam '%c' (%R)>", self->tag, f); + PyObject *result = PyUnicode_FromFormat("<cparam '%s' (%R)>", self->tag, f); Py_DECREF(f); return result; } case 'c': if (is_literal_char((unsigned char)self->value.c)) { - return PyUnicode_FromFormat("<cparam '%c' ('%c')>", + return PyUnicode_FromFormat("<cparam '%s' ('%c')>", self->tag, self->value.c); } else { - return PyUnicode_FromFormat("<cparam '%c' ('\\x%02x')>", + return PyUnicode_FromFormat("<cparam '%s' ('\\x%02x')>", self->tag, (unsigned char)self->value.c); } @@ -561,20 +567,16 @@ PyCArg_repr(PyObject *op) case 'z': case 'Z': case 'P': - return PyUnicode_FromFormat("<cparam '%c' (%p)>", + return PyUnicode_FromFormat("<cparam '%s' (%p)>", self->tag, self->value.p); - break; default: - if (is_literal_char((unsigned char)self->tag)) { - return PyUnicode_FromFormat("<cparam '%c' at %p>", - (unsigned char)self->tag, (void *)self); - } - else { - return PyUnicode_FromFormat("<cparam 0x%02x at %p>", - (unsigned char)self->tag, (void *)self); - } + break; } + +generic: + return PyUnicode_FromFormat("<cparam '%s' at %p>", + self->tag, (void *)self); } static PyMemberDef PyCArgType_members[] = { @@ -1807,7 +1809,7 @@ _ctypes_byref_impl(PyObject *module, PyObject *obj, Py_ssize_t offset) if (parg == NULL) return NULL; - parg->tag = 'P'; + parg->tag = "P"; parg->pffi_type = &ffi_type_pointer; parg->obj = Py_NewRef(obj); parg->value.p = (char *)((CDataObject *)obj)->b_ptr + offset; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7b6b7f08582251b..248559aa364a198 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -494,7 +494,7 @@ PyObject *_ctypes_callproc(ctypes_state *st, struct tagPyCArgObject { PyObject_HEAD ffi_type *pffi_type; - char tag; + const char *tag; union { char c; char b; @@ -511,7 +511,7 @@ struct tagPyCArgObject { long double G[2]; } value; PyObject *obj; - Py_ssize_t size; /* for the 'V' tag */ + Py_ssize_t size; /* for the "V" tag */ }; #define _PyCArgObject_CAST(op) ((PyCArgObject *)(op)) diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 83802605e1f4dc9..c192ce5f05452fe 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -360,17 +360,17 @@ _curses_panel_panel_bottom_impl(PyCursesPanelObject *self) } /*[clinic input] -@permit_long_docstring_body _curses_panel.panel.hide Hide the panel. -This does not delete the object, it just makes the window on screen invisible. +This does not delete the object, it just makes the window on screen +invisible. [clinic start generated code]*/ static PyObject * _curses_panel_panel_hide_impl(PyCursesPanelObject *self) -/*[clinic end generated code: output=a7bbbd523e1eab49 input=9071b463a39a1a6a]*/ +/*[clinic end generated code: output=a7bbbd523e1eab49 input=9456aca9b264dde1]*/ { int rtn = hide_panel(self->pan); return curses_panel_panel_check_err(self, rtn, "hide_panel", "hide"); @@ -380,11 +380,13 @@ _curses_panel_panel_hide_impl(PyCursesPanelObject *self) _curses_panel.panel.show Display the panel (which might have been hidden). + +The panel is placed on top of the panel stack. [clinic start generated code]*/ static PyObject * _curses_panel_panel_show_impl(PyCursesPanelObject *self) -/*[clinic end generated code: output=6b4553ab45c97769 input=57b167bbefaa3755]*/ +/*[clinic end generated code: output=6b4553ab45c97769 input=9997cf364ca71422]*/ { int rtn = show_panel(self->pan); return curses_panel_panel_check_err(self, rtn, "show_panel", "show"); @@ -670,7 +672,13 @@ static PyMethodDef PyCursesPanel_Methods[] = { /* -------------------------------------------------------*/ +PyDoc_STRVAR(PyCursesPanel_Type_doc, +"A curses panel.\n" +"\n" +"Panel objects are returned by new_panel()."); + static PyType_Slot PyCursesPanel_Type_slots[] = { + {Py_tp_doc, (void *)PyCursesPanel_Type_doc}, {Py_tp_clear, PyCursesPanel_Clear}, {Py_tp_dealloc, PyCursesPanel_Dealloc}, {Py_tp_traverse, PyCursesPanel_Traverse}, @@ -724,11 +732,13 @@ _curses_panel.new_panel / Return a panel object, associating it with the given window win. + +The new panel is placed on top of the panel stack. [clinic start generated code]*/ static PyObject * _curses_panel_new_panel_impl(PyObject *module, PyCursesWindowObject *win) -/*[clinic end generated code: output=45e948e0176a9bd2 input=74d4754e0ebe4800]*/ +/*[clinic end generated code: output=45e948e0176a9bd2 input=3b6fea647b808fd7]*/ { PANEL *pan = new_panel(win->win); if (pan == NULL) { @@ -772,12 +782,13 @@ _curses_panel.update_panels Updates the virtual screen after changes in the panel stack. -This does not call curses.doupdate(), so you'll have to do this yourself. +This does not call curses.doupdate(), so you'll have to do this +yourself. [clinic start generated code]*/ static PyObject * _curses_panel_update_panels_impl(PyObject *module) -/*[clinic end generated code: output=2f3b4c2e03d90ded input=5299624c9a708621]*/ +/*[clinic end generated code: output=2f3b4c2e03d90ded input=0d0db79f05ec3ef4]*/ { PyCursesInitialised; update_panels(); @@ -816,8 +827,10 @@ _curses_panel_exec(PyObject *mod) } /* For exception _curses_panel.error */ - state->error = PyErr_NewException( - "_curses_panel.error", NULL, NULL); + state->error = PyErr_NewExceptionWithDoc( + "_curses_panel.error", + "Exception raised when a curses panel library function returns an error.", + NULL, NULL); if (PyModule_AddObjectRef(mod, "error", state->error) < 0) { return -1; diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 000d7318557a6e6..3377367d4ce45df 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -41,12 +41,12 @@ Here's a list of currently unsupported functions: addchnstr addchstr color_set define_key - del_curterm delscreen dupwin inchnstr inchstr innstr keyok + del_curterm dupwin inchnstr inchstr innstr keyok mcprint mvaddchnstr mvaddchstr mvcur mvinchnstr mvinchstr mvinnstr mmvwaddchnstr mvwaddchstr - mvwinchnstr mvwinchstr mvwinnstr newterm + mvwinchnstr mvwinchstr mvwinnstr restartterm ripoffline scr_dump - scr_init scr_restore scr_set scrl set_curterm set_term setterm + scr_init scr_restore scr_set scrl set_curterm setterm tgetent tgetflag tgetnum tgetstr tgoto timeout tputs vidattr vidputs waddchnstr waddchstr wcolor_set winchnstr winchstr winnstr wmouse_trafo wscrl @@ -108,7 +108,7 @@ static const char PyCursesVersion[] = "2.2"; #include "pycore_capsule.h" // _PyCapsule_SetTraverse() #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_structseq.h" // _PyStructSequence_NewType() -#include "pycore_fileutils.h" // _Py_set_inheritable +#include "pycore_fileutils.h" // _Py_dup(), _Py_set_inheritable() #ifdef __hpux #define STRICT_SYSV_CURSES @@ -164,6 +164,9 @@ typedef chtype attr_t; /* No attr_t type is available */ typedef struct { PyObject *error; // curses exception type PyTypeObject *window_type; // exposed by PyCursesWindow_Type + PyTypeObject *screen_type; // _curses.screen + PyObject *topscreen; // owned ref to the current screen object, + // or NULL for the initscr() screen } cursesmodule_state; static inline cursesmodule_state * @@ -189,12 +192,14 @@ get_cursesmodule_state_by_win(PyCursesWindowObject *win) } #define _PyCursesWindowObject_CAST(op) ((PyCursesWindowObject *)(op)) +#define _PyCursesScreenObject_CAST(op) ((PyCursesScreenObject *)(op)) /*[clinic input] module _curses class _curses.window "PyCursesWindowObject *" "clinic_state()->window_type" +class _curses.screen "PyCursesScreenObject *" "clinic_state()->screen_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=ae6cb623018f2cbc]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b027ab105ab94e1]*/ /* Indicate whether the module has already been loaded or not. */ static int curses_module_loaded = 0; @@ -208,7 +213,11 @@ static int curses_initscr_called = FALSE; /* Tells whether start_color() has been called to initialise color usage. */ static int curses_start_color_called = FALSE; -static const char *curses_screen_encoding = NULL; +/* Encoding of the initial screen, used by module-level functions that have + no window object to take it from (e.g. unctrl(), ungetch()). This is a + private copy: the window object that initscr() returns may be deallocated + while these functions are still in use. */ +static char *curses_screen_encoding = NULL; /* Utility Error Procedures */ @@ -496,7 +505,8 @@ PyCurses_ConvertToChtype(PyCursesWindowObject *win, PyObject *obj, chtype *ch) - int - bytes of length 1 - - str of length 1 + - str of length 1, or a spacing character followed by up to + CCHARW_MAX - 1 combining characters Return: @@ -512,20 +522,43 @@ PyCurses_ConvertToCchar_t(PyCursesWindowObject *win, PyObject *obj, ) { long value; -#ifdef HAVE_NCURSESW - wchar_t buffer[2]; -#endif if (PyUnicode_Check(obj)) { #ifdef HAVE_NCURSESW - if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { + /* A character cell may hold a spacing character plus up to + CCHARW_MAX - 1 combining characters; wch must point to a buffer + of at least CCHARW_MAX + 1 wide characters. */ + Py_ssize_t nch = PyUnicode_AsWideChar(obj, wch, CCHARW_MAX + 1); + if (nch < 0) { + return 0; + } + if (nch == 0 || nch > CCHARW_MAX) { PyErr_Format(PyExc_TypeError, - "expect int or bytes or str of length 1, " - "got a str of length %zi", - PyUnicode_GET_LENGTH(obj)); + "expect int or bytes or a string of 1 to %d " + "characters, got a str of length %zi", + (int)CCHARW_MAX, PyUnicode_GET_LENGTH(obj)); return 0; } - *wch = buffer[0]; + /* A character cell is a single spacing character optionally followed + by combining characters. A lone control character is still allowed + (like addch(ord('\n'))), but in a multi-character cell the base must + be a printable character and the rest must be zero-width combining + characters. Validate this explicitly: otherwise setcchar() would + silently drop a trailing spacing character, or fail with a generic + error for a control character used as the base. */ + if (nch > 1) { + int bad = wcwidth(wch[0]) < 0; + for (Py_ssize_t i = 1; !bad && i < nch; i++) { + bad = wcwidth(wch[i]) != 0; + } + if (bad) { + PyErr_SetString(PyExc_ValueError, + "a character cell must be a single spacing " + "character optionally followed by combining " + "characters"); + return 0; + } + } return 2; #else return PyCurses_ConvertToChtype(win, obj, ch); @@ -613,6 +646,33 @@ PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj, return 0; } +#ifdef HAVE_NCURSESW +/* Build a single character cell from obj. + + On success return 1 and store the raw chtype (without *attr*) in *pch when + obj is an int or bytes, or return 2 and store a cchar_t (with *attr* + applied) in *pwc when obj is a str -- a spacing character optionally + followed by combining characters. Return 0 and set an exception on error. + + This lets a method use the wide *_set functions (which accept combining + characters) for string arguments while still accepting integer chtype + values. */ +static int +PyCurses_ConvertToCell(PyCursesWindowObject *win, PyObject *obj, long attr, + const char *funcname, chtype *pch, cchar_t *pwc) +{ + wchar_t wstr[CCHARW_MAX + 1]; + int type = PyCurses_ConvertToCchar_t(win, obj, pch, wstr); + if (type == 2) { + if (setcchar(pwc, wstr, (attr_t)attr, PAIR_NUMBER(attr), NULL) == ERR) { + curses_window_set_error(win, "setcchar", funcname); + return 0; + } + } + return type; +} +#endif + static int color_allow_default_converter(PyObject *arg, void *ptr) { @@ -840,6 +900,52 @@ Window_NoArgNoReturnFunction(wdeleteln) Window_NoArgTrueFalseFunction(is_wintouched) +#if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20110404 +Window_NoArgTrueFalseFunction(is_cleared) +Window_NoArgTrueFalseFunction(is_idcok) +Window_NoArgTrueFalseFunction(is_idlok) +Window_NoArgTrueFalseFunction(is_immedok) +Window_NoArgTrueFalseFunction(is_keypad) +Window_NoArgTrueFalseFunction(is_leaveok) +Window_NoArgTrueFalseFunction(is_nodelay) +Window_NoArgTrueFalseFunction(is_notimeout) +Window_NoArgTrueFalseFunction(is_pad) +Window_NoArgTrueFalseFunction(is_scrollok) +Window_NoArgTrueFalseFunction(is_subwin) +Window_NoArgTrueFalseFunction(is_syncok) + +static PyObject * +PyCursesWindow_getdelay(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + return PyLong_FromLong(wgetdelay(self->win)); +} + +static PyObject * +PyCursesWindow_getscrreg(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int top, bottom; + if (wgetscrreg(self->win, &top, &bottom) == ERR) { + curses_window_set_error(self, "wgetscrreg", "getscrreg"); + return NULL; + } + return Py_BuildValue("(ii)", top, bottom); +} + +static PyObject * +PyCursesWindow_getparent(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + /* The standard window has no parent; subwindows keep a reference to the + window they were derived from. */ + if (self->orig == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef((PyObject *)self->orig); +} +#endif /* NCURSES_EXT_FUNCS */ + Window_NoArgNoReturnVoidFunction(wsyncup) Window_NoArgNoReturnVoidFunction(wsyncdown) Window_NoArgNoReturnVoidFunction(wstandend) @@ -884,7 +990,7 @@ Window_TwoArgNoReturnFunction(wresize, int, "ii;lines,columns") static PyObject * PyCursesWindow_New(cursesmodule_state *state, WINDOW *win, const char *encoding, - PyCursesWindowObject *orig) + PyCursesWindowObject *orig, PyObject *screen) { if (encoding == NULL) { #if defined(MS_WINDOWS) @@ -912,14 +1018,14 @@ PyCursesWindow_New(cursesmodule_state *state, return NULL; } wo->win = win; + wo->orig = (PyCursesWindowObject *)Py_XNewRef((PyObject *)orig); + wo->screen = Py_XNewRef(screen); wo->encoding = _PyMem_Strdup(encoding); if (wo->encoding == NULL) { Py_DECREF(wo); PyErr_NoMemory(); return NULL; } - wo->orig = orig; - Py_XINCREF(orig); PyObject_GC_Track((PyObject *)wo); return (PyObject *)wo; } @@ -940,6 +1046,7 @@ PyCursesWindow_dealloc(PyObject *self) PyMem_Free(wo->encoding); } Py_XDECREF(wo->orig); + Py_XDECREF(wo->screen); window_type->tp_free(self); Py_DECREF(window_type); } @@ -950,6 +1057,7 @@ PyCursesWindow_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); PyCursesWindowObject *wo = (PyCursesWindowObject *)self; Py_VISIT(wo->orig); + Py_VISIT(wo->screen); return 0; } @@ -993,7 +1101,7 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, int type; chtype cch = 0; #ifdef HAVE_NCURSESW - wchar_t wstr[2]; + wchar_t wstr[CCHARW_MAX + 1]; cchar_t wcval; #endif const char *funcname; @@ -1001,7 +1109,6 @@ _curses_window_addch_impl(PyCursesWindowObject *self, int group_left_1, #ifdef HAVE_NCURSESW type = PyCurses_ConvertToCchar_t(self, ch, &cch, wstr); if (type == 2) { - wstr[1] = L'\0'; rtn = setcchar(&wcval, wstr, attr, PAIR_NUMBER(attr), NULL); if (rtn == ERR) { curses_window_set_error(self, "setcchar", "addch"); @@ -1273,11 +1380,22 @@ _curses_window_bkgd_impl(PyCursesWindowObject *self, PyObject *ch, long attr) /*[clinic end generated code: output=058290afb2cf4034 input=634015bcb339283d]*/ { chtype bkgd; - +#ifdef HAVE_NCURSESW + cchar_t wch; + int type = PyCurses_ConvertToCell(self, ch, attr, "bkgd", &bkgd, &wch); + if (type == 0) { + return NULL; + } + if (type == 2) { + int rtn = wbkgrnd(self->win, &wch); + return curses_window_check_err(self, rtn, "wbkgrnd", "bkgd"); + } +#else if (!PyCurses_ConvertToChtype(self, ch, &bkgd)) return NULL; +#endif - int rtn = wbkgd(self->win, bkgd | attr); + int rtn = wbkgd(self->win, bkgd | (attr_t)attr); return curses_window_check_err(self, rtn, "wbkgd", "bkgd"); } @@ -1350,16 +1468,26 @@ _curses_window_bkgdset_impl(PyCursesWindowObject *self, PyObject *ch, /*[clinic end generated code: output=8cb994fc4d7e2496 input=e09c682425c9e45b]*/ { chtype bkgd; - +#ifdef HAVE_NCURSESW + cchar_t wch; + int type = PyCurses_ConvertToCell(self, ch, attr, "bkgdset", &bkgd, &wch); + if (type == 0) { + return NULL; + } + if (type == 2) { + wbkgrndset(self->win, &wch); + Py_RETURN_NONE; + } +#else if (!PyCurses_ConvertToChtype(self, ch, &bkgd)) return NULL; +#endif - wbkgdset(self->win, bkgd | attr); + wbkgdset(self->win, bkgd | (attr_t)attr); Py_RETURN_NONE; } /*[clinic input] -@permit_long_docstring_body _curses.window.border ls: object(c_default="NULL") = _curses.ACS_VLINE @@ -1382,10 +1510,10 @@ _curses.window.border Draw a border around the edges of the window. -Each parameter specifies the character to use for a specific part of the -border. The characters can be specified as integers or as one-character -strings. A 0 value for any parameter will cause the default character to be -used for that parameter. +Each parameter specifies the character to use for a specific part of +the border. The characters can be specified as integers or as +one-character strings. A 0 value for any parameter will cause the +default character to be used for that parameter. [clinic start generated code]*/ static PyObject * @@ -1393,29 +1521,60 @@ _curses_window_border_impl(PyCursesWindowObject *self, PyObject *ls, PyObject *rs, PyObject *ts, PyObject *bs, PyObject *tl, PyObject *tr, PyObject *bl, PyObject *br) -/*[clinic end generated code: output=670ef38d3d7c2aa3 input=adaafca87488ee35]*/ +/*[clinic end generated code: output=670ef38d3d7c2aa3 input=42568c1458221d24]*/ { chtype ch[8]; int i, rtn; + PyObject *objs[8] = {ls, rs, ts, bs, tl, tr, bl, br}; /* Clear the array of parameters */ - for(i=0; i<8; i++) + for (i = 0; i < 8; i++) ch[i] = 0; -#define CONVERTTOCHTYPE(obj, i) \ - if ((obj) != NULL && !PyCurses_ConvertToChtype(self, (obj), &ch[(i)])) \ - return NULL; - - CONVERTTOCHTYPE(ls, 0); - CONVERTTOCHTYPE(rs, 1); - CONVERTTOCHTYPE(ts, 2); - CONVERTTOCHTYPE(bs, 3); - CONVERTTOCHTYPE(tl, 4); - CONVERTTOCHTYPE(tr, 5); - CONVERTTOCHTYPE(bl, 6); - CONVERTTOCHTYPE(br, 7); - -#undef CONVERTTOCHTYPE +#ifdef HAVE_NCURSESW + cchar_t wch[8]; + const cchar_t *wch_p[8]; + int use_wide = 0; + int types[8]; + for (i = 0; i < 8; i++) { + types[i] = 0; + if (objs[i] != NULL) { + types[i] = PyCurses_ConvertToCell(self, objs[i], A_NORMAL, + "border", &ch[i], &wch[i]); + if (types[i] == 0) { + return NULL; + } + if (types[i] == 2) { + use_wide = 1; + } + } + } + if (use_wide) { + for (i = 0; i < 8; i++) { + if (objs[i] == NULL) { + wch_p[i] = NULL; /* use the default character */ + } + else if (types[i] == 2) { + wch_p[i] = &wch[i]; + } + else { + PyErr_SetString(PyExc_TypeError, + "border() cannot mix integer or bytes " + "characters with wide string characters"); + return NULL; + } + } + rtn = wborder_set(self->win, + wch_p[0], wch_p[1], wch_p[2], wch_p[3], + wch_p[4], wch_p[5], wch_p[6], wch_p[7]); + return curses_window_check_err(self, rtn, "wborder_set", "border"); + } +#else + for (i = 0; i < 8; i++) { + if (objs[i] != NULL && !PyCurses_ConvertToChtype(self, objs[i], &ch[i])) + return NULL; + } +#endif rtn = wborder(self->win, ch[0], ch[1], ch[2], ch[3], @@ -1436,16 +1595,42 @@ _curses.window.box Draw a border around the edges of the window. -Similar to border(), but both ls and rs are verch and both ts and bs are -horch. The default corner characters are always used by this function. +Similar to border(), but both ls and rs are verch and both ts and bs +are horch. The default corner characters are always used by this +function. [clinic start generated code]*/ static PyObject * _curses_window_box_impl(PyCursesWindowObject *self, int group_right_1, PyObject *verch, PyObject *horch) -/*[clinic end generated code: output=f3fcb038bb287192 input=f00435f9c8c98f60]*/ +/*[clinic end generated code: output=f3fcb038bb287192 input=e11acb7dbf6790b6]*/ { chtype ch1 = 0, ch2 = 0; +#ifdef HAVE_NCURSESW + cchar_t wch1, wch2; + int t1 = 0, t2 = 0; + if (group_right_1) { + t1 = PyCurses_ConvertToCell(self, verch, A_NORMAL, "box", &ch1, &wch1); + if (t1 == 0) { + return NULL; + } + t2 = PyCurses_ConvertToCell(self, horch, A_NORMAL, "box", &ch2, &wch2); + if (t2 == 0) { + return NULL; + } + } + if (t1 == 2 || t2 == 2) { + if (t1 != 2 || t2 != 2) { + PyErr_SetString(PyExc_TypeError, + "box() cannot mix integer or bytes characters " + "with wide string characters"); + return NULL; + } + int rtn = wborder_set(self->win, &wch1, &wch1, &wch2, &wch2, + NULL, NULL, NULL, NULL); + return curses_window_check_err(self, rtn, "wborder_set", "box"); + } +#else if (group_right_1) { if (!PyCurses_ConvertToChtype(self, verch, &ch1)) { return NULL; @@ -1454,6 +1639,7 @@ _curses_window_box_impl(PyCursesWindowObject *self, int group_right_1, return NULL; } } +#endif return curses_window_check_err(self, box(self->win, ch1, ch2), "box", NULL); } @@ -1574,13 +1760,16 @@ _curses.window.delch ] / -Delete any character at (y, x). +Delete the character under the cursor, or at (y, x) if specified. + +All characters to the right on the same line are shifted one +position left. [clinic start generated code]*/ static PyObject * _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=22e77bb9fa11b461 input=d2f79e630a4fc6d0]*/ +/*[clinic end generated code: output=22e77bb9fa11b461 input=61db3c7f4885e90c]*/ { int rtn; const char *funcname; @@ -1596,7 +1785,6 @@ _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.derwin [ @@ -1613,15 +1801,15 @@ _curses.window.derwin Create a sub-window (window-relative coordinates). -derwin() is the same as calling subwin(), except that begin_y and begin_x -are relative to the origin of the window, rather than relative to the entire -screen. +derwin() is the same as calling subwin(), except that begin_y and +begin_x are relative to the origin of the window, rather than +relative to the entire screen. [clinic start generated code]*/ static PyObject * _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=7924b112d9f70d6e input=ebe95ded1c284c8e]*/ +/*[clinic end generated code: output=7924b112d9f70d6e input=6efb50722be444ba]*/ { WINDOW *win; @@ -1633,7 +1821,7 @@ _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, } cursesmodule_state *state = get_cursesmodule_state_by_win(self); - return PyCursesWindow_New(state, win, NULL, self); + return PyCursesWindow_New(state, win, NULL, self, self->screen); } /*[clinic input] @@ -1655,9 +1843,32 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, /*[clinic end generated code: output=13e7dd875d4b9642 input=e7f34b964e92b156]*/ { chtype ch_; - +#ifdef HAVE_NCURSESW + cchar_t wch; + int type = PyCurses_ConvertToCell(self, ch, attr, "echochar", &ch_, &wch); + if (type == 0) { + return NULL; + } + if (type == 2) { + int rtn; + const char *funcname; +#ifdef py_is_pad + if (py_is_pad(self->win)) { + rtn = pecho_wchar(self->win, &wch); + funcname = "pecho_wchar"; + } + else +#endif + { + rtn = wecho_wchar(self->win, &wch); + funcname = "wecho_wchar"; + } + return curses_window_check_err(self, rtn, funcname, "echochar"); + } +#else if (!PyCurses_ConvertToChtype(self, ch, &ch_)) return NULL; +#endif int rtn; const char *funcname; @@ -1731,7 +1942,6 @@ curses_check_signals_on_input_error(PyCursesWindowObject *self, } /*[clinic input] -@permit_long_docstring_body _curses.window.getch [ @@ -1744,15 +1954,16 @@ _curses.window.getch Get a character code from terminal keyboard. -The integer returned does not have to be in ASCII range: function keys, -keypad keys and so on return numbers higher than 256. In no-delay mode, -1 -is returned if there is no input, else getch() waits until a key is pressed. +The integer returned does not have to be in ASCII range: function +keys, keypad keys and so on return numbers higher than 256. In +no-delay mode, -1 is returned if there is no input, else getch() +waits until a key is pressed. [clinic start generated code]*/ static PyObject * _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=e1639e87d545e676 input=9a053077373e2a30]*/ +/*[clinic end generated code: output=e1639e87d545e676 input=0dc5ff40e079787a]*/ { int rtn; @@ -1779,7 +1990,6 @@ _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.getkey [ @@ -1792,15 +2002,16 @@ _curses.window.getkey Get a character (string) from terminal keyboard. -Returning a string instead of an integer, as getch() does. Function keys, -keypad keys and other special keys return a multibyte string containing the -key name. In no-delay mode, an exception is raised if there is no input. +Returning a string instead of an integer, as getch() does. Function +keys, keypad keys and other special keys return a multibyte string +containing the key name. In no-delay mode, an exception is raised +if there is no input. [clinic start generated code]*/ static PyObject * _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=8490a182db46b10f input=5177f03fb6c31ea6]*/ +/*[clinic end generated code: output=8490a182db46b10f input=bd24a7da1ed9c73b]*/ { int rtn; @@ -2007,21 +2218,33 @@ _curses_window_hline_impl(PyCursesWindowObject *self, int group_left_1, /*[clinic end generated code: output=c00d489d61fc9eef input=81a4dea47268163e]*/ { chtype ch_; - +#ifdef HAVE_NCURSESW + cchar_t wch; + int type = PyCurses_ConvertToCell(self, ch, attr, "hline", &ch_, &wch); + if (type == 0) { + return NULL; + } +#else if (!PyCurses_ConvertToChtype(self, ch, &ch_)) return NULL; +#endif if (group_left_1) { if (wmove(self->win, y, x) == ERR) { curses_window_set_error(self, "wmove", "hline"); return NULL; } } +#ifdef HAVE_NCURSESW + if (type == 2) { + int rtn = whline_set(self->win, &wch, n); + return curses_window_check_err(self, rtn, "whline_set", "hline"); + } +#endif int rtn = whline(self->win, ch_ | (attr_t)attr, n); return curses_window_check_err(self, rtn, "whline", "hline"); } /*[clinic input] -@permit_long_docstring_body _curses.window.insch [ @@ -2042,23 +2265,41 @@ _curses.window.insch Insert a character before the current or specified position. -All characters to the right of the cursor are shifted one position right, with -the rightmost characters on the line being lost. +All characters to the right of the cursor are shifted one position +right, with the rightmost characters on the line being lost. [clinic start generated code]*/ static PyObject * _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *ch, int group_right_1, long attr) -/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=3f2a230cb09fed5a]*/ +/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=d662a0f96f33e15a]*/ { int rtn; chtype ch_ = 0; - + const char *funcname; +#ifdef HAVE_NCURSESW + cchar_t wch; + int type = PyCurses_ConvertToCell(self, ch, attr, "insch", &ch_, &wch); + if (type == 0) { + return NULL; + } + if (type == 2) { + if (!group_left_1) { + rtn = wins_wch(self->win, &wch); + funcname = "wins_wch"; + } + else { + rtn = mvwins_wch(self->win, y, x, &wch); + funcname = "mvwins_wch"; + } + return curses_window_check_err(self, rtn, funcname, "insch"); + } +#else if (!PyCurses_ConvertToChtype(self, ch, &ch_)) return NULL; +#endif - const char *funcname; if (!group_left_1) { rtn = winsch(self->win, ch_ | (attr_t)attr); funcname = "winsch"; @@ -2072,7 +2313,6 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.inch [ @@ -2085,13 +2325,14 @@ _curses.window.inch Return the character at the given position in the window. -The bottom 8 bits are the character proper, and upper bits are the attributes. +The bottom 8 bits are the character proper, and upper bits are the +attributes. [clinic start generated code]*/ static PyObject * _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=97ca8581baaafd06 input=a5846f315464dc86]*/ +/*[clinic end generated code: output=97ca8581baaafd06 input=7a03956d94dc9a69]*/ { chtype rtn; const char *funcname; @@ -2122,10 +2363,12 @@ PyDoc_STRVAR(_curses_window_instr__doc__, " n\n" " Maximal number of characters.\n" "\n" -"Return a string of characters, extracted from the window starting at the\n" -"current cursor position, or at y, x if specified. Attributes are stripped\n" -"from the characters. If n is specified, instr() returns a string at most\n" -"n characters long (exclusive of the trailing NUL)."); +"Return a string of characters, extracted from the window starting\n" +"at the current cursor position, or at y, x if specified, and\n" +"stopping at the end of the line. Attributes and color\n" +"information are stripped from the characters. If n is specified,\n" +"instr() returns a string at most n characters long (exclusive of\n" +"the trailing NUL)."); static PyObject * PyCursesWindow_instr(PyObject *op, PyObject *args) @@ -2162,6 +2405,125 @@ PyCursesWindow_instr(PyObject *op, PyObject *args) return PyBytesWriter_FinishWithSize(writer, strlen(buf)); } +#ifdef HAVE_NCURSESW +PyDoc_STRVAR(_curses_window_get_wstr__doc__, +"get_wstr([[y, x,] n=2047])\n" +"Read a string from the user, with primitive line editing capacity.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Maximal number of characters.\n" +"\n" +"This is the wide-character variant of getstr(); it returns a str."); + +static PyObject * +PyCursesWindow_get_wstr(PyObject *op, PyObject *args) +{ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn, use_xy = 0, y = 0, x = 0; + unsigned int max_buf_size = 2048; + unsigned int n = max_buf_size - 1; + + if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, + "_curses.window.get_wstr")) + { + return NULL; + } + + n = Py_MIN(n, max_buf_size - 1); + wint_t *buf = PyMem_New(wint_t, n + 1); + if (buf == NULL) { + return PyErr_NoMemory(); + } + + if (use_xy) { + Py_BEGIN_ALLOW_THREADS + rtn = mvwgetn_wstr(self->win, y, x, buf, n); + Py_END_ALLOW_THREADS + } + else { + Py_BEGIN_ALLOW_THREADS + rtn = wgetn_wstr(self->win, buf, n); + Py_END_ALLOW_THREADS + } + + if (rtn == ERR) { + PyMem_Free(buf); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } + + /* wgetn_wstr() fills a wint_t buffer; copy it to a wchar_t buffer. */ + Py_ssize_t len = 0; + while (buf[len]) { + len++; + } + wchar_t *wbuf = PyMem_New(wchar_t, len + 1); + if (wbuf == NULL) { + PyMem_Free(buf); + return PyErr_NoMemory(); + } + for (Py_ssize_t i = 0; i < len; i++) { + wbuf[i] = (wchar_t)buf[i]; + } + PyObject *res = PyUnicode_FromWideChar(wbuf, len); + PyMem_Free(wbuf); + PyMem_Free(buf); + return res; +} + +PyDoc_STRVAR(_curses_window_in_wstr__doc__, +"in_wstr([y, x,] n=2047)\n" +"Return a string of characters, extracted from the window.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Maximal number of characters.\n" +"\n" +"This is the wide-character variant of instr(); it returns a str."); + +static PyObject * +PyCursesWindow_in_wstr(PyObject *op, PyObject *args) +{ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn, use_xy = 0, y = 0, x = 0; + unsigned int max_buf_size = 2048; + unsigned int n = max_buf_size - 1; + + if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, + "_curses.window.in_wstr")) + { + return NULL; + } + + n = Py_MIN(n, max_buf_size - 1); + wchar_t *buf = PyMem_New(wchar_t, n + 1); + if (buf == NULL) { + return PyErr_NoMemory(); + } + + if (use_xy) { + rtn = mvwinnwstr(self->win, y, x, buf, n); + } + else { + rtn = winnwstr(self->win, buf, n); + } + + if (rtn == ERR) { + PyMem_Free(buf); + return Py_GetConstant(Py_CONSTANT_EMPTY_STR); + } + PyObject *res = PyUnicode_FromWideChar(buf, -1); + PyMem_Free(buf); + return res; +} +#endif /* HAVE_NCURSESW */ + /*[clinic input] _curses.window.insstr @@ -2183,18 +2545,18 @@ _curses.window.insstr Insert the string before the current or specified position. -Insert a character string (as many characters as will fit on the line) -before the character under the cursor. All characters to the right of -the cursor are shifted right, with the rightmost characters on the line -being lost. The cursor position does not change (after moving to y, x, -if specified). +Insert a character string (as many characters as will fit on the +line) before the character under the cursor. All characters to the +right of the cursor are shifted right, with the rightmost characters +on the line being lost. The cursor position does not change (after +moving to y, x, if specified). [clinic start generated code]*/ static PyObject * _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int group_right_1, long attr) -/*[clinic end generated code: output=c259a5265ad0b777 input=6827cddc6340a7f3]*/ +/*[clinic end generated code: output=c259a5265ad0b777 input=dbfbdd3892155ea6]*/ { int rtn; int strtype; @@ -2260,7 +2622,6 @@ _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.insnstr [ @@ -2284,19 +2645,19 @@ _curses.window.insnstr Insert at most n characters of the string. -Insert a character string (as many characters as will fit on the line) -before the character under the cursor, up to n characters. If n is zero -or negative, the entire string is inserted. All characters to the right -of the cursor are shifted right, with the rightmost characters on the line -being lost. The cursor position does not change (after moving to y, x, if -specified). +Insert a character string (as many characters as will fit on the +line) before the character under the cursor, up to n characters. If +n is zero or negative, the entire string is inserted. All +characters to the right of the cursor are shifted right, with the +rightmost characters on the line being lost. The cursor position +does not change (after moving to y, x, if specified). [clinic start generated code]*/ static PyObject * _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int n, int group_right_1, long attr) -/*[clinic end generated code: output=971a32ea6328ec8b input=dcdc554102fbcd5d]*/ +/*[clinic end generated code: output=971a32ea6328ec8b input=fd0a9b65b84b385f]*/ { int rtn; int strtype; @@ -2361,7 +2722,7 @@ _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary _curses.window.is_linetouched line: int @@ -2370,12 +2731,13 @@ _curses.window.is_linetouched Return True if the specified line was modified, otherwise return False. -Raise a curses.error exception if line is not valid for the given window. +Raise a curses.error exception if line is not valid for the given +window. [clinic start generated code]*/ static PyObject * _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line) -/*[clinic end generated code: output=ad4a4edfee2db08c input=af71c040b951c467]*/ +/*[clinic end generated code: output=ad4a4edfee2db08c input=18924dfac25ab7f1]*/ { int erg; erg = is_linetouched(self->win, line); @@ -2388,7 +2750,6 @@ _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line) #ifdef py_is_pad /*[clinic input] -@permit_long_docstring_body _curses.window.noutrefresh [ @@ -2403,9 +2764,9 @@ _curses.window.noutrefresh Mark for refresh but wait. -This function updates the data structure representing the desired state of the -window, but does not force an update of the physical screen. To accomplish -that, call doupdate(). +This function updates the data structure representing the desired +state of the window, but does not force an update of the physical +screen. To accomplish that, call doupdate(). [clinic start generated code]*/ static PyObject * @@ -2413,22 +2774,21 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=809a1f3c6a03e23e input=b39fe8fc79b9980b]*/ +/*[clinic end generated code: output=809a1f3c6a03e23e input=8b4c74bf55008803]*/ #else /*[clinic input] -@permit_long_docstring_body _curses.window.noutrefresh Mark for refresh but wait. -This function updates the data structure representing the desired state of the -window, but does not force an update of the physical screen. To accomplish -that, call doupdate(). +This function updates the data structure representing the desired +state of the window, but does not force an update of the physical +screen. To accomplish that, call doupdate(). [clinic start generated code]*/ static PyObject * _curses_window_noutrefresh_impl(PyCursesWindowObject *self) -/*[clinic end generated code: output=6ef6dec666643fee input=6a9f59ae5e4c139e]*/ +/*[clinic end generated code: output=6ef6dec666643fee input=a7c6306f8af9d0dd]*/ #endif { int rtn; @@ -2461,7 +2821,6 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) } /*[clinic input] -@permit_long_docstring_body _curses.window.overlay destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") @@ -2478,14 +2837,15 @@ _curses.window.overlay Overlay the window on top of destwin. -The windows need not be the same size, only the overlapping region is copied. -This copy is non-destructive, which means that the current background -character does not overwrite the old contents of destwin. +The windows need not be the same size, only the overlapping region +is copied. This copy is non-destructive, which means that the +current background character does not overwrite the old contents of +destwin. -To get fine-grained control over the copied region, the second form of -overlay() can be used. sminrow and smincol are the upper-left coordinates -of the source window, and the other variables mark a rectangle in the -destination window. +To get fine-grained control over the copied region, the second form +of overlay() can be used. sminrow and smincol are the upper-left +coordinates of the source window, and the other variables mark +a rectangle in the destination window. [clinic start generated code]*/ static PyObject * @@ -2493,7 +2853,7 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, PyCursesWindowObject *destwin, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=82bb2c4cb443ca58 input=dd6af34deb892a65]*/ +/*[clinic end generated code: output=82bb2c4cb443ca58 input=da0cec7f7bda1b3f]*/ { int rtn; @@ -2509,7 +2869,6 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, } /*[clinic input] -@permit_long_docstring_body _curses.window.overwrite destwin: object(type="PyCursesWindowObject *", subclass_of="clinic_state()->window_type") @@ -2526,14 +2885,15 @@ _curses.window.overwrite Overwrite the window on top of destwin. -The windows need not be the same size, in which case only the overlapping -region is copied. This copy is destructive, which means that the current -background character overwrites the old contents of destwin. +The windows need not be the same size, in which case only the +overlapping region is copied. This copy is destructive, which means +that the current background character overwrites the old contents of +destwin. -To get fine-grained control over the copied region, the second form of -overwrite() can be used. sminrow and smincol are the upper-left coordinates -of the source window, the other variables mark a rectangle in the destination -window. +To get fine-grained control over the copied region, the second form +of overwrite() can be used. sminrow and smincol are the upper-left +coordinates of the source window, the other variables mark +a rectangle in the destination window. [clinic start generated code]*/ static PyObject * @@ -2542,7 +2902,7 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=12ae007d1681be28 input=e84d8ebdf1c09596]*/ +/*[clinic end generated code: output=12ae007d1681be28 input=4244ab8a97087898]*/ { int rtn; @@ -2558,6 +2918,7 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, } /*[clinic input] +@permit_long_summary _curses.window.putwin file: object @@ -2570,7 +2931,7 @@ This information can be later retrieved using the getwin() function. static PyObject * _curses_window_putwin_impl(PyCursesWindowObject *self, PyObject *file) -/*[clinic end generated code: output=fdae68ac59b0281b input=0608648e09c8ea0a]*/ +/*[clinic end generated code: output=fdae68ac59b0281b input=959fc85a9e4a31c2]*/ { /* We have to simulate this by writing to a temporary FILE*, then reading back, then writing to the argument file. */ @@ -2626,7 +2987,6 @@ _curses_window_redrawln_impl(PyCursesWindowObject *self, int beg, int num) } /*[clinic input] -@permit_long_docstring_body _curses.window.refresh [ @@ -2642,23 +3002,24 @@ _curses.window.refresh Update the display immediately. Synchronize actual screen with previous drawing/deleting methods. -The 6 optional arguments can only be specified when the window is a pad -created with newpad(). The additional parameters are needed to indicate -what part of the pad and screen are involved. pminrow and pmincol specify -the upper left-hand corner of the rectangle to be displayed in the pad. -sminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to -be displayed on the screen. The lower right-hand corner of the rectangle to -be displayed in the pad is calculated from the screen coordinates, since the -rectangles must be the same size. Both rectangles must be entirely contained -within their respective structures. Negative values of pminrow, pmincol, -sminrow, or smincol are treated as if they were zero. +The 6 optional arguments can only be specified when the window is +a pad created with newpad(). The additional parameters are needed +to indicate what part of the pad and screen are involved. pminrow +and pmincol specify the upper left-hand corner of the rectangle to +be displayed in the pad. sminrow, smincol, smaxrow, and smaxcol +specify the edges of the rectangle to be displayed on the screen. +The lower right-hand corner of the rectangle to be displayed in the +pad is calculated from the screen coordinates, since the rectangles +must be the same size. Both rectangles must be entirely contained +within their respective structures. Negative values of pminrow, +pmincol, sminrow, or smincol are treated as if they were zero. [clinic start generated code]*/ static PyObject * _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=42199543115e6e63 input=65405c03290496a6]*/ +/*[clinic end generated code: output=42199543115e6e63 input=ff2e900c6b2696b1]*/ { int rtn; @@ -2711,7 +3072,6 @@ _curses_window_setscrreg_impl(PyCursesWindowObject *self, int top, } /*[clinic input] -@permit_long_docstring_body _curses.window.subwin [ @@ -2728,14 +3088,14 @@ _curses.window.subwin Create a sub-window (screen-relative coordinates). -By default, the sub-window will extend from the specified position to the -lower right corner of the window. +By default, the sub-window will extend from the specified position +to the lower right corner of the window. [clinic start generated code]*/ static PyObject * _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=93e898afc348f59a input=5292cf610e2f3585]*/ +/*[clinic end generated code: output=93e898afc348f59a input=07b5058cb8820595]*/ { WINDOW *win; const char *funcname; @@ -2759,11 +3119,10 @@ _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, } cursesmodule_state *state = get_cursesmodule_state_by_win(self); - return PyCursesWindow_New(state, win, self->encoding, self); + return PyCursesWindow_New(state, win, self->encoding, self, self->screen); } /*[clinic input] -@permit_long_docstring_body _curses.window.scroll [ @@ -2774,13 +3133,14 @@ _curses.window.scroll Scroll the screen or scrolling region. -Scroll upward if the argument is positive and downward if it is negative. +Scroll upward if the argument is positive and downward if it is +negative. [clinic start generated code]*/ static PyObject * _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, int lines) -/*[clinic end generated code: output=4541a8a11852d360 input=386456524c550113]*/ +/*[clinic end generated code: output=4541a8a11852d360 input=d8d81a5b52b9b40f]*/ { int rtn; const char *funcname; @@ -2796,7 +3156,6 @@ _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, } /*[clinic input] -@permit_long_docstring_body _curses.window.touchline start: int @@ -2808,14 +3167,15 @@ _curses.window.touchline Pretend count lines have been changed, starting with line start. -If changed is supplied, it specifies whether the affected lines are marked -as having been changed (changed=True) or unchanged (changed=False). +If changed is supplied, it specifies whether the affected lines are +marked as having been changed (changed=True) or unchanged +(changed=False). [clinic start generated code]*/ static PyObject * _curses_window_touchline_impl(PyCursesWindowObject *self, int start, int count, int group_right_1, int changed) -/*[clinic end generated code: output=65d05b3f7438c61d input=36e13b6f5eb591f5]*/ +/*[clinic end generated code: output=65d05b3f7438c61d input=e0dc62f90d9dea55]*/ { int rtn; const char *funcname; @@ -2861,15 +3221,28 @@ _curses_window_vline_impl(PyCursesWindowObject *self, int group_left_1, /*[clinic end generated code: output=287ad1cc8982217f input=a6f2dc86a4648b32]*/ { chtype ch_; - +#ifdef HAVE_NCURSESW + cchar_t wch; + int type = PyCurses_ConvertToCell(self, ch, attr, "vline", &ch_, &wch); + if (type == 0) { + return NULL; + } +#else if (!PyCurses_ConvertToChtype(self, ch, &ch_)) return NULL; +#endif if (group_left_1) { if (wmove(self->win, y, x) == ERR) { curses_window_set_error(self, "wmove", "vline"); return NULL; } } +#ifdef HAVE_NCURSESW + if (type == 2) { + int rtn = wvline_set(self->win, &wch, n); + return curses_window_check_err(self, rtn, "wvline_set", "vline"); + } +#endif int rtn = wvline(self->win, ch_ | (attr_t)attr, n); return curses_window_check_err(self, rtn, "wvline", "vline"); } @@ -2919,6 +3292,84 @@ PyCursesWindow_set_encoding(PyObject *op, PyObject *value, void *Py_UNUSED(ignor #include "clinic/_cursesmodule.c.h" #undef clinic_state +#if defined(HAVE_CURSES_USE_SCREEN) || defined(HAVE_CURSES_USE_WINDOW) +/* Shared trampoline for window.use()/screen.use(): call + func(obj, *extra, **kwargs) and store the result (NULL on exception) in + data->result. */ +typedef struct { + PyObject *obj; /* the window or screen object */ + PyObject *func; /* the callable */ + PyObject *extra; /* extra positional arguments (a tuple) */ + PyObject *kwargs; /* keyword arguments (a dict), or NULL */ + PyObject *result; /* output: the call result, or NULL */ +} curses_use_data; + +static void +curses_use_call(curses_use_data *data) +{ + Py_ssize_t n = PyTuple_GET_SIZE(data->extra); + PyObject *callargs = PyTuple_New(n + 1); + if (callargs == NULL) { + data->result = NULL; + return; + } + PyTuple_SET_ITEM(callargs, 0, Py_NewRef(data->obj)); + for (Py_ssize_t i = 0; i < n; i++) { + PyTuple_SET_ITEM(callargs, i + 1, + Py_NewRef(PyTuple_GET_ITEM(data->extra, i))); + } + data->result = PyObject_Call(data->func, callargs, data->kwargs); + Py_DECREF(callargs); +} + +/* Parse (func, *extra) from a use() method's argument tuple. */ +static int +curses_use_parse(PyObject *args, PyObject **func, PyObject **extra) +{ + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + if (nargs < 1) { + PyErr_SetString(PyExc_TypeError, + "use() missing required argument 'func'"); + return -1; + } + *func = PyTuple_GET_ITEM(args, 0); + if (!PyCallable_Check(*func)) { + PyErr_SetString(PyExc_TypeError, "use(): func must be callable"); + return -1; + } + *extra = PyTuple_GetSlice(args, 1, nargs); + return *extra == NULL ? -1 : 0; +} +#endif + +#ifdef HAVE_CURSES_USE_WINDOW +static int +curses_use_window_cb(WINDOW *Py_UNUSED(win), void *data) +{ + curses_use_call((curses_use_data *)data); + return 0; +} + +PyDoc_STRVAR(PyCursesWindow_use__doc__, +"use($self, func, /, *args, **kwargs)\n--\n\n" +"Call func(win, *args, **kwargs) with the window locked,\n" +"and return its result."); + +static PyObject * +PyCursesWindow_use(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyCursesWindowObject *wo = _PyCursesWindowObject_CAST(self); + PyObject *func, *extra; + if (curses_use_parse(args, &func, &extra) < 0) { + return NULL; + } + curses_use_data data = {self, func, extra, kwargs, NULL}; + use_window(wo->win, curses_use_window_cb, &data); + Py_DECREF(extra); + return data.result; +} +#endif /* HAVE_CURSES_USE_WINDOW */ + static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ADDCH_METHODDEF _CURSES_WINDOW_ADDNSTR_METHODDEF @@ -2936,81 +3387,221 @@ static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_BKGDSET_METHODDEF _CURSES_WINDOW_BORDER_METHODDEF _CURSES_WINDOW_BOX_METHODDEF - {"clear", PyCursesWindow_wclear, METH_NOARGS}, - {"clearok", PyCursesWindow_clearok, METH_VARARGS}, - {"clrtobot", PyCursesWindow_wclrtobot, METH_NOARGS}, - {"clrtoeol", PyCursesWindow_wclrtoeol, METH_NOARGS}, - {"cursyncup", PyCursesWindow_wcursyncup, METH_NOARGS}, + {"clear", PyCursesWindow_wclear, METH_NOARGS, + "clear($self, /)\n--\n\n" + "Clear the window and repaint it completely on the next refresh()."}, + {"clearok", PyCursesWindow_clearok, METH_VARARGS, + "clearok($self, flag, /)\n--\n\n" + "Clear the window on the next refresh() if flag is true."}, + {"clrtobot", PyCursesWindow_wclrtobot, METH_NOARGS, + "clrtobot($self, /)\n--\n\n" + "Erase from the cursor to the end of the window."}, + {"clrtoeol", PyCursesWindow_wclrtoeol, METH_NOARGS, + "clrtoeol($self, /)\n--\n\n" + "Erase from the cursor to the end of the line."}, + {"cursyncup", PyCursesWindow_wcursyncup, METH_NOARGS, + "cursyncup($self, /)\n--\n\n" + "Update the cursor position of all ancestor windows to match."}, _CURSES_WINDOW_DELCH_METHODDEF - {"deleteln", PyCursesWindow_wdeleteln, METH_NOARGS}, + {"deleteln", PyCursesWindow_wdeleteln, METH_NOARGS, + "deleteln($self, /)\n--\n\n" + "Delete the line under the cursor; move following lines up by one."}, _CURSES_WINDOW_DERWIN_METHODDEF _CURSES_WINDOW_ECHOCHAR_METHODDEF _CURSES_WINDOW_ENCLOSE_METHODDEF - {"erase", PyCursesWindow_werase, METH_NOARGS}, - {"getbegyx", PyCursesWindow_getbegyx, METH_NOARGS}, + {"erase", PyCursesWindow_werase, METH_NOARGS, + "erase($self, /)\n--\n\n" + "Clear the window."}, + {"getbegyx", PyCursesWindow_getbegyx, METH_NOARGS, + "getbegyx($self, /)\n--\n\n" + "Return a tuple (y, x) of the upper-left corner coordinates."}, _CURSES_WINDOW_GETBKGD_METHODDEF _CURSES_WINDOW_GETCH_METHODDEF _CURSES_WINDOW_GETKEY_METHODDEF _CURSES_WINDOW_GET_WCH_METHODDEF - {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS}, - {"getparyx", PyCursesWindow_getparyx, METH_NOARGS}, - { - "getstr", PyCursesWindow_getstr, METH_VARARGS, +#if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20110404 + {"getdelay", PyCursesWindow_getdelay, METH_NOARGS, + "getdelay($self, /)\n--\n\n" + "Return the window's read timeout in milliseconds.\n\n" + "-1 means blocking, 0 means non-blocking; see nodelay() and timeout()."}, +#endif + {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS, + "getmaxyx($self, /)\n--\n\n" + "Return a tuple (y, x) of the window height and width."}, +#if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20110404 + {"getparent", PyCursesWindow_getparent, METH_NOARGS, + "getparent($self, /)\n--\n\n" + "Return the parent window, or None if this is not a subwindow."}, +#endif + {"getparyx", PyCursesWindow_getparyx, METH_NOARGS, + "getparyx($self, /)\n--\n\n" + "Return (y, x) relative to the parent window, or (-1, -1) if none."}, +#if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20110404 + {"getscrreg", PyCursesWindow_getscrreg, METH_NOARGS, + "getscrreg($self, /)\n--\n\n" + "Return a tuple (top, bottom) of the current scrolling region."}, +#endif + { + "getstr", PyCursesWindow_getstr, METH_VARARGS, _curses_window_getstr__doc__ }, - {"getyx", PyCursesWindow_getyx, METH_NOARGS}, +#ifdef HAVE_NCURSESW + { + "get_wstr", PyCursesWindow_get_wstr, METH_VARARGS, + _curses_window_get_wstr__doc__ + }, +#endif + {"getyx", PyCursesWindow_getyx, METH_NOARGS, + "getyx($self, /)\n--\n\n" + "Return a tuple (y, x) of the current cursor position."}, _CURSES_WINDOW_HLINE_METHODDEF - {"idcok", PyCursesWindow_idcok, METH_VARARGS}, - {"idlok", PyCursesWindow_idlok, METH_VARARGS}, + {"idcok", PyCursesWindow_idcok, METH_VARARGS, + "idcok($self, flag, /)\n--\n\n" + "Enable or disable the hardware insert/delete character feature."}, + {"idlok", PyCursesWindow_idlok, METH_VARARGS, + "idlok($self, flag, /)\n--\n\n" + "Enable or disable the hardware insert/delete line feature."}, #ifdef HAVE_CURSES_IMMEDOK - {"immedok", PyCursesWindow_immedok, METH_VARARGS}, + {"immedok", PyCursesWindow_immedok, METH_VARARGS, + "immedok($self, flag, /)\n--\n\n" + "If flag is true, refresh the window on every change to it."}, #endif _CURSES_WINDOW_INCH_METHODDEF _CURSES_WINDOW_INSCH_METHODDEF - {"insdelln", PyCursesWindow_winsdelln, METH_VARARGS}, - {"insertln", PyCursesWindow_winsertln, METH_NOARGS}, + {"insdelln", PyCursesWindow_winsdelln, METH_VARARGS, + "insdelln($self, nlines, /)\n--\n\n" + "Insert (nlines > 0) or delete (nlines < 0) lines above the cursor."}, + {"insertln", PyCursesWindow_winsertln, METH_NOARGS, + "insertln($self, /)\n--\n\n" + "Insert a blank line under the cursor; move following lines down."}, _CURSES_WINDOW_INSNSTR_METHODDEF _CURSES_WINDOW_INSSTR_METHODDEF { "instr", PyCursesWindow_instr, METH_VARARGS, _curses_window_instr__doc__ }, +#ifdef HAVE_NCURSESW + { + "in_wstr", PyCursesWindow_in_wstr, METH_VARARGS, + _curses_window_in_wstr__doc__ + }, +#endif _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF - {"is_wintouched", PyCursesWindow_is_wintouched, METH_NOARGS}, - {"keypad", PyCursesWindow_keypad, METH_VARARGS}, - {"leaveok", PyCursesWindow_leaveok, METH_VARARGS}, - {"move", PyCursesWindow_wmove, METH_VARARGS}, - {"mvderwin", PyCursesWindow_mvderwin, METH_VARARGS}, - {"mvwin", PyCursesWindow_mvwin, METH_VARARGS}, - {"nodelay", PyCursesWindow_nodelay, METH_VARARGS}, - {"notimeout", PyCursesWindow_notimeout, METH_VARARGS}, + {"is_wintouched", PyCursesWindow_is_wintouched, METH_NOARGS, + "is_wintouched($self, /)\n--\n\n" + "Return True if the window changed since the last refresh()."}, +#if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20110404 + {"is_cleared", PyCursesWindow_is_cleared, METH_NOARGS, + "is_cleared($self, /)\n--\n\n" + "Return the current value set by clearok()."}, + {"is_idcok", PyCursesWindow_is_idcok, METH_NOARGS, + "is_idcok($self, /)\n--\n\n" + "Return the current value set by idcok()."}, + {"is_idlok", PyCursesWindow_is_idlok, METH_NOARGS, + "is_idlok($self, /)\n--\n\n" + "Return the current value set by idlok()."}, + {"is_immedok", PyCursesWindow_is_immedok, METH_NOARGS, + "is_immedok($self, /)\n--\n\n" + "Return the current value set by immedok()."}, + {"is_keypad", PyCursesWindow_is_keypad, METH_NOARGS, + "is_keypad($self, /)\n--\n\n" + "Return the current value set by keypad()."}, + {"is_leaveok", PyCursesWindow_is_leaveok, METH_NOARGS, + "is_leaveok($self, /)\n--\n\n" + "Return the current value set by leaveok()."}, + {"is_nodelay", PyCursesWindow_is_nodelay, METH_NOARGS, + "is_nodelay($self, /)\n--\n\n" + "Return the current value set by nodelay()."}, + {"is_notimeout", PyCursesWindow_is_notimeout, METH_NOARGS, + "is_notimeout($self, /)\n--\n\n" + "Return the current value set by notimeout()."}, + {"is_pad", PyCursesWindow_is_pad, METH_NOARGS, + "is_pad($self, /)\n--\n\n" + "Return True if the window is a pad."}, + {"is_scrollok", PyCursesWindow_is_scrollok, METH_NOARGS, + "is_scrollok($self, /)\n--\n\n" + "Return the current value set by scrollok()."}, + {"is_subwin", PyCursesWindow_is_subwin, METH_NOARGS, + "is_subwin($self, /)\n--\n\n" + "Return True if the window is a subwindow."}, + {"is_syncok", PyCursesWindow_is_syncok, METH_NOARGS, + "is_syncok($self, /)\n--\n\n" + "Return the current value set by syncok()."}, +#endif + {"keypad", PyCursesWindow_keypad, METH_VARARGS, + "keypad($self, flag, /)\n--\n\n" + "Interpret escape sequences for special keys if flag is true."}, + {"leaveok", PyCursesWindow_leaveok, METH_VARARGS, + "leaveok($self, flag, /)\n--\n\n" + "If flag is true, leave the cursor where the update leaves it."}, + {"move", PyCursesWindow_wmove, METH_VARARGS, + "move($self, new_y, new_x, /)\n--\n\n" + "Move the cursor to (new_y, new_x)."}, + {"mvderwin", PyCursesWindow_mvderwin, METH_VARARGS, + "mvderwin($self, y, x, /)\n--\n\n" + "Move the window inside its parent window."}, + {"mvwin", PyCursesWindow_mvwin, METH_VARARGS, + "mvwin($self, new_y, new_x, /)\n--\n\n" + "Move the window so its upper-left corner is at (new_y, new_x)."}, + {"nodelay", PyCursesWindow_nodelay, METH_VARARGS, + "nodelay($self, flag, /)\n--\n\n" + "If flag is true, getch() becomes non-blocking."}, + {"notimeout", PyCursesWindow_notimeout, METH_VARARGS, + "notimeout($self, flag, /)\n--\n\n" + "If flag is true, do not time out escape sequences."}, _CURSES_WINDOW_NOUTREFRESH_METHODDEF _CURSES_WINDOW_OVERLAY_METHODDEF _CURSES_WINDOW_OVERWRITE_METHODDEF _CURSES_WINDOW_PUTWIN_METHODDEF _CURSES_WINDOW_REDRAWLN_METHODDEF - {"redrawwin", PyCursesWindow_redrawwin, METH_NOARGS}, + {"redrawwin", PyCursesWindow_redrawwin, METH_NOARGS, + "redrawwin($self, /)\n--\n\n" + "Mark the entire window for redraw on the next refresh()."}, _CURSES_WINDOW_REFRESH_METHODDEF #ifndef STRICT_SYSV_CURSES - {"resize", PyCursesWindow_wresize, METH_VARARGS}, + {"resize", PyCursesWindow_wresize, METH_VARARGS, + "resize($self, nlines, ncols, /)\n--\n\n" + "Resize the window to nlines rows and ncols columns."}, #endif _CURSES_WINDOW_SCROLL_METHODDEF - {"scrollok", PyCursesWindow_scrollok, METH_VARARGS}, + {"scrollok", PyCursesWindow_scrollok, METH_VARARGS, + "scrollok($self, flag, /)\n--\n\n" + "Control whether the window scrolls when the cursor moves off it."}, _CURSES_WINDOW_SETSCRREG_METHODDEF - {"standend", PyCursesWindow_wstandend, METH_NOARGS}, - {"standout", PyCursesWindow_wstandout, METH_NOARGS}, + {"standend", PyCursesWindow_wstandend, METH_NOARGS, + "standend($self, /)\n--\n\n" + "Turn off the standout attribute."}, + {"standout", PyCursesWindow_wstandout, METH_NOARGS, + "standout($self, /)\n--\n\n" + "Turn on the A_STANDOUT attribute."}, {"subpad", _curses_window_subwin, METH_VARARGS, _curses_window_subwin__doc__}, _CURSES_WINDOW_SUBWIN_METHODDEF - {"syncdown", PyCursesWindow_wsyncdown, METH_NOARGS}, + {"syncdown", PyCursesWindow_wsyncdown, METH_NOARGS, + "syncdown($self, /)\n--\n\n" + "Touch each location changed in any ancestor of the window."}, #ifdef HAVE_CURSES_SYNCOK - {"syncok", PyCursesWindow_syncok, METH_VARARGS}, + {"syncok", PyCursesWindow_syncok, METH_VARARGS, + "syncok($self, flag, /)\n--\n\n" + "If flag is true, call syncup() on every change to the window."}, #endif - {"syncup", PyCursesWindow_wsyncup, METH_NOARGS}, - {"timeout", PyCursesWindow_wtimeout, METH_VARARGS}, + {"syncup", PyCursesWindow_wsyncup, METH_NOARGS, + "syncup($self, /)\n--\n\n" + "Touch locations in ancestors that changed in this window."}, + {"timeout", PyCursesWindow_wtimeout, METH_VARARGS, + "timeout($self, delay, /)\n--\n\n" + "Set blocking or non-blocking read behavior for the window."}, _CURSES_WINDOW_TOUCHLINE_METHODDEF - {"touchwin", PyCursesWindow_touchwin, METH_NOARGS}, - {"untouchwin", PyCursesWindow_untouchwin, METH_NOARGS}, + {"touchwin", PyCursesWindow_touchwin, METH_NOARGS, + "touchwin($self, /)\n--\n\n" + "Mark the whole window as changed."}, + {"untouchwin", PyCursesWindow_untouchwin, METH_NOARGS, + "untouchwin($self, /)\n--\n\n" + "Mark all lines in the window as unchanged since last refresh()."}, _CURSES_WINDOW_VLINE_METHODDEF +#ifdef HAVE_CURSES_USE_WINDOW + {"use", _PyCFunction_CAST(PyCursesWindow_use), + METH_VARARGS | METH_KEYWORDS, PyCursesWindow_use__doc__}, +#endif {NULL, NULL} /* sentinel */ }; @@ -3024,7 +3615,14 @@ static PyGetSetDef PyCursesWindow_getsets[] = { {NULL, NULL, NULL, NULL } /* sentinel */ }; +PyDoc_STRVAR(PyCursesWindow_Type_doc, +"A curses window.\n" +"\n" +"Window objects are returned by initscr() and newwin(), and by the\n" +"methods that create subwindows and pads."); + static PyType_Slot PyCursesWindow_Type_slots[] = { + {Py_tp_doc, (void *)PyCursesWindow_Type_doc}, {Py_tp_methods, PyCursesWindow_methods}, {Py_tp_getset, PyCursesWindow_getsets}, {Py_tp_dealloc, PyCursesWindow_dealloc}, @@ -3043,6 +3641,182 @@ static PyType_Spec PyCursesWindow_Type_spec = { .slots = PyCursesWindow_Type_slots }; +/* -------------------------------------------------------*/ +/* Screen objects (multiple terminals) */ +/* -------------------------------------------------------*/ + +static PyObject * +PyCursesScreen_New(cursesmodule_state *state, SCREEN *screen, + FILE *outfp, FILE *infp, PyObject *stdscr) +{ + PyCursesScreenObject *so = PyObject_GC_New(PyCursesScreenObject, + state->screen_type); + if (so == NULL) { + return NULL; + } + so->screen = screen; + so->outfp = outfp; + so->infp = infp; + so->stdscr = Py_XNewRef(stdscr); + PyObject_GC_Track((PyObject *)so); + return (PyObject *)so; +} + +/* Free the C SCREEN and the FILE* streams owned by a screen object. + Safe to call more than once. + + This must run by reference counting (from the dealloc), not from tp_clear: + it has to happen only once every window on the screen is gone, and thus + after del_panel() for any panel built on one of those windows. delscreen() + tears down the screen that del_panel() needs, so a panel outliving its + screen would crash. */ +static void +curses_screen_close(PyCursesScreenObject *so) +{ + if (so->screen != NULL) { + delscreen(so->screen); + so->screen = NULL; + } + if (so->outfp != NULL) { + fclose(so->outfp); + so->outfp = NULL; + } + if (so->infp != NULL) { + fclose(so->infp); + so->infp = NULL; + } +} + +static PyObject * +PyCursesScreen_get_stdscr(PyObject *self, void *Py_UNUSED(closure)) +{ + PyCursesScreenObject *so = _PyCursesScreenObject_CAST(self); + if (so->stdscr == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(so->stdscr); +} + +static int +PyCursesScreen_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(_PyCursesScreenObject_CAST(self)->stdscr); + return 0; +} + +static int +PyCursesScreen_clear(PyObject *self) +{ + PyCursesScreenObject *so = _PyCursesScreenObject_CAST(self); + /* Break the reference cycle between a screen and its standard window by + dropping the reference to that window. Do NOT delscreen() here: that is + deferred to the dealloc so it runs after every window (see + curses_screen_close()). delscreen() will free the standard window, so + detach it from its wrapper first: the wrapper must not delwin() a window + that delscreen() frees. Any further use of the wrapper operates on a + NULL window and fails cleanly. */ + if (so->stdscr != NULL) { + ((PyCursesWindowObject *)so->stdscr)->win = NULL; + } + Py_CLEAR(so->stdscr); + return 0; +} + +static void +PyCursesScreen_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + (void)PyCursesScreen_clear(self); + curses_screen_close(_PyCursesScreenObject_CAST(self)); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyGetSetDef PyCursesScreen_getsets[] = { + {"stdscr", PyCursesScreen_get_stdscr, NULL, + "the screen's standard window (stdscr)", NULL}, + {NULL, NULL, NULL, NULL, NULL} /* sentinel */ +}; + +#ifdef HAVE_CURSES_USE_SCREEN +static int +curses_use_screen_cb(SCREEN *Py_UNUSED(sp), void *data) +{ + curses_use_call((curses_use_data *)data); + return 0; +} + +PyDoc_STRVAR(PyCursesScreen_use__doc__, +"use($self, func, /, *args, **kwargs)\n--\n\n" +"Call func(screen, *args, **kwargs) with the screen locked,\n" +"and return its result."); + +static PyObject * +PyCursesScreen_use(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyCursesScreenObject *so = _PyCursesScreenObject_CAST(self); + if (so->screen == NULL) { + cursesmodule_state *state = get_cursesmodule_state_by_cls(Py_TYPE(self)); + PyErr_SetString(state->error, "the screen has been deleted"); + return NULL; + } + PyObject *func, *extra; + if (curses_use_parse(args, &func, &extra) < 0) { + return NULL; + } + curses_use_data data = {self, func, extra, kwargs, NULL}; + use_screen(so->screen, curses_use_screen_cb, &data); + Py_DECREF(extra); + return data.result; +} +#endif /* HAVE_CURSES_USE_SCREEN */ + +PyDoc_STRVAR(PyCursesScreen_close__doc__, +"close($self, /)\n--\n\n" +"Detach the screen's standard window, breaking their reference cycle.\n\n" +"Afterwards the stdscr attribute is None and the window it returned earlier\n" +"can no longer be used. The screen is released once it and its windows are\n" +"no longer referenced."); + +static PyObject * +PyCursesScreen_close(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + (void)PyCursesScreen_clear(self); + Py_RETURN_NONE; +} + +static PyMethodDef PyCursesScreen_methods[] = { + {"close", PyCursesScreen_close, METH_NOARGS, + PyCursesScreen_close__doc__}, +#ifdef HAVE_CURSES_USE_SCREEN + {"use", _PyCFunction_CAST(PyCursesScreen_use), + METH_VARARGS | METH_KEYWORDS, PyCursesScreen_use__doc__}, +#endif + {NULL, NULL} /* sentinel */ +}; + +static PyType_Slot PyCursesScreen_Type_slots[] = { + {Py_tp_methods, PyCursesScreen_methods}, + {Py_tp_getset, PyCursesScreen_getsets}, + {Py_tp_dealloc, PyCursesScreen_dealloc}, + {Py_tp_traverse, PyCursesScreen_traverse}, + {Py_tp_clear, PyCursesScreen_clear}, + {0, NULL} +}; + +static PyType_Spec PyCursesScreen_Type_spec = { + .name = "_curses.screen", + .basicsize = sizeof(PyCursesScreenObject), + .flags = Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HEAPTYPE + | Py_TPFLAGS_HAVE_GC, + .slots = PyCursesScreen_Type_slots +}; + /* -------------------------------------------------------*/ /* @@ -3126,11 +3900,17 @@ static PyType_Spec PyCursesWindow_Type_spec = { /*[clinic input] _curses.filter +Restrict screen updates to the current line. + +Must be called before initscr(). Afterwards curses confines the cursor +and screen updates to a single line, which is useful for enabling +character-at-a-time line editing without touching the rest of the +screen. [clinic start generated code]*/ static PyObject * _curses_filter_impl(PyObject *module) -/*[clinic end generated code: output=fb5b8a3642eb70b5 input=668c75a6992d3624]*/ +/*[clinic end generated code: output=fb5b8a3642eb70b5 input=e3c64d6ab2106132]*/ { /* not checking for PyCursesInitialised here since filter() must be called before initscr() */ @@ -3139,6 +3919,28 @@ _curses_filter_impl(PyObject *module) } #endif +#ifdef HAVE_CURSES_NOFILTER +/*[clinic input] +_curses.nofilter + +Undo the effect of a preceding filter() call. + +Must be called before initscr(). It restores the normal behaviour that +filter() disables, so that the next initscr() or newterm() uses the full +screen rather than a single line. +[clinic start generated code]*/ + +static PyObject * +_curses_nofilter_impl(PyObject *module) +/*[clinic end generated code: output=d95ca4d48a6bdbdf input=53183055c0901ab7]*/ +{ + /* not checking for PyCursesInitialised here since nofilter() must + be called before initscr() */ + nofilter(); + Py_RETURN_NONE; +} +#endif + /*[clinic input] _curses.baudrate @@ -3182,20 +3984,79 @@ _curses.cbreak Enter cbreak mode. -In cbreak mode (sometimes called "rare" mode) normal tty line buffering is -turned off and characters are available to be read one by one. However, -unlike raw mode, special characters (interrupt, quit, suspend, and flow -control) retain their effects on the tty driver and calling program. -Calling first raw() then cbreak() leaves the terminal in cbreak mode. +In cbreak mode (sometimes called "rare" mode) normal tty line buffering +is turned off and characters are available to be read one by one. +However, unlike raw mode, special characters (interrupt, quit, suspend, +and flow control) retain their effects on the tty driver and calling +program. Calling first raw() then cbreak() leaves the terminal in +cbreak mode. [clinic start generated code]*/ static PyObject * _curses_cbreak_impl(PyObject *module, int flag) -/*[clinic end generated code: output=9f9dee9664769751 input=c7d0bddda93016c1]*/ +/*[clinic end generated code: output=9f9dee9664769751 input=42d81687f11ddbf3]*/ NoArgOrFlagNoReturnFunctionBody(cbreak, flag) +/* is_cbreak()/is_echo()/is_nl()/is_raw() were added in ncurses 6.5. */ +#if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427 +/*[clinic input] +_curses.is_cbreak + +Return True if cbreak mode is enabled, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_curses_is_cbreak_impl(PyObject *module) +/*[clinic end generated code: output=8a1ad7889fb43daf input=99988df6fd2f1c81]*/ +{ + PyCursesStatefulInitialised(module); + return PyBool_FromLong(is_cbreak()); +} + +/*[clinic input] +_curses.is_echo + +Return True if echo mode is enabled, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_curses_is_echo_impl(PyObject *module) +/*[clinic end generated code: output=72692d2aa41591c4 input=f6152cf7c00e47eb]*/ +{ + PyCursesStatefulInitialised(module); + return PyBool_FromLong(is_echo()); +} + +/*[clinic input] +_curses.is_nl + +Return True if nl mode is enabled, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_curses_is_nl_impl(PyObject *module) +/*[clinic end generated code: output=999eb44abc43ce65 input=1e0a2607e45a01e1]*/ +{ + PyCursesStatefulInitialised(module); + return PyBool_FromLong(is_nl()); +} + +/*[clinic input] +_curses.is_raw + +Return True if raw mode is enabled, False otherwise. +[clinic start generated code]*/ + +static PyObject * +_curses_is_raw_impl(PyObject *module) +/*[clinic end generated code: output=dd9816d777561c35 input=a64fa6a251ed3ece]*/ +{ + PyCursesStatefulInitialised(module); + return PyBool_FromLong(is_raw()); +} +#endif /* NCURSES_EXT_FUNCS */ + /*[clinic input] -@permit_long_docstring_body _curses.color_content color_number: color @@ -3204,13 +4065,14 @@ _curses.color_content Return the red, green, and blue (RGB) components of the specified color. -A 3-tuple is returned, containing the R, G, B values for the given color, -which will be between 0 (no component) and 1000 (maximum amount of component). +A 3-tuple is returned, containing the R, G, B values for the given +color, which will be between 0 (no component) and 1000 (maximum amount +of component). [clinic start generated code]*/ static PyObject * _curses_color_content_impl(PyObject *module, int color_number) -/*[clinic end generated code: output=17b466df7054e0de input=baffe25b351eb916]*/ +/*[clinic end generated code: output=17b466df7054e0de input=c95fb50093fa0be0]*/ { _CURSES_COLOR_VAL_TYPE r,g,b; @@ -3236,12 +4098,13 @@ _curses.color_pair Return the attribute value for displaying text in the specified color. This attribute value can be combined with A_STANDOUT, A_REVERSE, and the -other A_* attributes. pair_number() is the counterpart to this function. +other A_* attributes. pair_number() is the counterpart to this +function. [clinic start generated code]*/ static PyObject * _curses_color_pair_impl(PyObject *module, int pair_number) -/*[clinic end generated code: output=60718abb10ce9feb input=6034e9146f343802]*/ +/*[clinic end generated code: output=60718abb10ce9feb input=cf74bb81d3cc3370]*/ { PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); @@ -3259,14 +4122,14 @@ _curses.curs_set Set the cursor state. If the terminal supports the visibility requested, the previous cursor -state is returned; otherwise, an exception is raised. On many terminals, -the "visible" mode is an underline cursor and the "very visible" mode is -a block cursor. +state is returned; otherwise, an exception is raised. On many +terminals, the "visible" mode is an underline cursor and the "very +visible" mode is a block cursor. [clinic start generated code]*/ static PyObject * _curses_curs_set_impl(PyObject *module, int visibility) -/*[clinic end generated code: output=ee8e62483b1d6cd4 input=81a7924a65d29504]*/ +/*[clinic end generated code: output=ee8e62483b1d6cd4 input=e010767a328f322b]*/ { int erg; @@ -3301,14 +4164,15 @@ _curses.def_shell_mode Save the current terminal mode as the "shell" mode. -The "shell" mode is the mode when the running program is not using curses. +The "shell" mode is the mode when the running program is not using +curses. Subsequent calls to reset_shell_mode() will restore this mode. [clinic start generated code]*/ static PyObject * _curses_def_shell_mode_impl(PyObject *module) -/*[clinic end generated code: output=d6e42f5c768f860f input=5ead21f6f0baa894]*/ +/*[clinic end generated code: output=d6e42f5c768f860f input=3809f85615c0b693]*/ NoArgNoReturnFunctionBody(def_shell_mode) /*[clinic input] @@ -3350,12 +4214,13 @@ _curses.echo Enter echo mode. -In echo mode, each character input is echoed to the screen as it is entered. +In echo mode, each character input is echoed to the screen as it is +entered. [clinic start generated code]*/ static PyObject * _curses_echo_impl(PyObject *module, int flag) -/*[clinic end generated code: output=03acb2ddfa6c8729 input=86cd4d5bb1d569c0]*/ +/*[clinic end generated code: output=03acb2ddfa6c8729 input=b4e9064326da9da4]*/ NoArgOrFlagNoReturnFunctionBody(echo, flag) /*[clinic input] @@ -3367,7 +4232,18 @@ De-initialize the library, and return terminal to normal status. static PyObject * _curses_endwin_impl(PyObject *module) /*[clinic end generated code: output=c0150cd96d2f4128 input=e172cfa43062f3fa]*/ -NoArgNoReturnFunctionBody(endwin) +{ + PyCursesStatefulInitialised(module); + + /* endwin() writes to the terminal and may call tcdrain(), which can block + (e.g. on a pty whose output is not being read); release the GIL so other + threads -- including one draining that terminal -- can run meanwhile. */ + int code; + Py_BEGIN_ALLOW_THREADS + code = endwin(); + Py_END_ALLOW_THREADS + return curses_check_err(module, code, "endwin", NULL); +} /*[clinic input] _curses.erasechar @@ -3388,18 +4264,41 @@ _curses_erasechar_impl(PyObject *module) return PyBytes_FromStringAndSize(&ch, 1); } +#ifdef HAVE_NCURSESW +/*[clinic input] +_curses.erasewchar + +Return the user's current wide-character erase character. +[clinic start generated code]*/ + +static PyObject * +_curses_erasewchar_impl(PyObject *module) +/*[clinic end generated code: output=7f3bd8c9097ac456 input=f7e9a3893b4df2f8]*/ +{ + wchar_t ch; + + PyCursesStatefulInitialised(module); + + if (erasewchar(&ch) == ERR) { + curses_set_error(module, "erasewchar", NULL); + return NULL; + } + return PyUnicode_FromWideChar(&ch, 1); +} +#endif /* HAVE_NCURSESW */ + /*[clinic input] -@permit_long_docstring_body _curses.flash Flash the screen. -That is, change it to reverse-video and then change it back in a short interval. +That is, change it to reverse-video and then change it back in a short +interval. [clinic start generated code]*/ static PyObject * _curses_flash_impl(PyObject *module) -/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=dd33d718e6edf436]*/ +/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=90878e305432add9]*/ NoArgNoReturnFunctionBody(flash) /*[clinic input] @@ -3407,13 +4306,13 @@ _curses.flushinp Flush all input buffers. -This throws away any typeahead that has been typed by the user and has not -yet been processed by the program. +This throws away any typeahead that has been typed by the user and has +not yet been processed by the program. [clinic start generated code]*/ static PyObject * _curses_flushinp_impl(PyObject *module) -/*[clinic end generated code: output=7e7a1fc1473960f5 input=59d042e705cef5ec]*/ +/*[clinic end generated code: output=7e7a1fc1473960f5 input=3a63c7213be8043c]*/ NoArgNoReturnVoidFunctionBody(flushinp) #ifdef getsyx @@ -3559,7 +4458,7 @@ _curses_getwin(PyObject *module, PyObject *file) goto error; } cursesmodule_state *state = get_cursesmodule_state(module); - res = PyCursesWindow_New(state, win, NULL, NULL); + res = PyCursesWindow_New(state, win, NULL, NULL, state->topscreen); error: fclose(fp); @@ -3599,6 +4498,7 @@ _curses_has_colors_impl(PyObject *module) NoArgTrueFalseFunctionBody(has_colors) /*[clinic input] +@permit_long_summary _curses.has_ic Return True if the terminal has insert- and delete-character capabilities. @@ -3606,7 +4506,7 @@ Return True if the terminal has insert- and delete-character capabilities. static PyObject * _curses_has_ic_impl(PyObject *module) -/*[clinic end generated code: output=6be24da9cb1268fe input=9bc2d3a797cc7324]*/ +/*[clinic end generated code: output=6be24da9cb1268fe input=e37fa080d879f7a9]*/ NoArgTrueFalseFunctionBody(has_ic) /*[clinic input] @@ -3622,6 +4522,7 @@ NoArgTrueFalseFunctionBody(has_il) #ifdef HAVE_CURSES_HAS_KEY /*[clinic input] +@permit_long_summary _curses.has_key key: int @@ -3633,7 +4534,7 @@ Return True if the current terminal type recognizes a key with that value. static PyObject * _curses_has_key_impl(PyObject *module, int key) -/*[clinic end generated code: output=19ad48319414d0b1 input=78bd44acf1a4997c]*/ +/*[clinic end generated code: output=19ad48319414d0b1 input=046ac6c72bbc9587]*/ { PyCursesStatefulInitialised(module); @@ -3688,13 +4589,14 @@ _curses.init_pair Change the definition of a color-pair. -If the color-pair was previously initialized, the screen is refreshed and -all occurrences of that color-pair are changed to the new definition. +If the color-pair was previously initialized, the screen is refreshed +and all occurrences of that color-pair are changed to the new +definition. [clinic start generated code]*/ static PyObject * _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) -/*[clinic end generated code: output=a0bba03d2bbc3ee6 input=54b421b44c12c389]*/ +/*[clinic end generated code: output=a0bba03d2bbc3ee6 input=5486c3a105130dae]*/ { PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); @@ -3715,42 +4617,125 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) Py_RETURN_NONE; } +#if _NCURSES_EXTENDED_COLOR_FUNCS /*[clinic input] -_curses.initscr +_curses.alloc_pair -Initialize the library. + fg: color_allow_default + Foreground color number. + bg: color_allow_default + Background color number. + / -Return a WindowObject which represents the whole screen. +Allocate a color pair for the given foreground and background colors. + +If a color pair for the same colors already exists, return its number. +Otherwise allocate a new color pair and return its number. [clinic start generated code]*/ static PyObject * -_curses_initscr_impl(PyObject *module) -/*[clinic end generated code: output=619fb68443810b7b input=514f4bce1821f6b5]*/ +_curses_alloc_pair_impl(PyObject *module, int fg, int bg) +/*[clinic end generated code: output=6eb08cb643d4b5a2 input=b29bafd7b360fa35]*/ { - WINDOW *win; + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); - if (curses_initscr_called) { - cursesmodule_state *state = get_cursesmodule_state(module); - int code = wrefresh(stdscr); - if (code == ERR) { - _curses_set_null_error(state, "wrefresh", "initscr"); - return NULL; - } - return PyCursesWindow_New(state, stdscr, NULL, NULL); + int pair = alloc_pair(fg, bg); + if (pair < 0) { + curses_set_error(module, "alloc_pair", NULL); + return NULL; } + return PyLong_FromLong(pair); +} - win = initscr(); +/*[clinic input] +_curses.find_pair - if (win == NULL) { - curses_set_null_error(module, "initscr", NULL); - return NULL; - } + fg: color_allow_default + Foreground color number. + bg: color_allow_default + Background color number. + / - curses_initscr_called = curses_setupterm_called = TRUE; +Return the number of a color pair for the given colors, or -1. + +Return -1 if no color pair for this combination of foreground and +background colors has been allocated. +[clinic start generated code]*/ + +static PyObject * +_curses_find_pair_impl(PyObject *module, int fg, int bg) +/*[clinic end generated code: output=376026c2a3ac4a9b input=930feac14892c251]*/ +{ + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); + + return PyLong_FromLong(find_pair(fg, bg)); +} + +/*[clinic input] +_curses.free_pair + + pair: pair + The number of the color pair to free. + / + +Free a color pair allocated by alloc_pair(). +[clinic start generated code]*/ + +static PyObject * +_curses_free_pair_impl(PyObject *module, int pair) +/*[clinic end generated code: output=61be0fb2e4bb4e4a input=d24df62feb4161c6]*/ +{ + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); + + return curses_check_err(module, free_pair(pair), "free_pair", NULL); +} + +/*[clinic input] +_curses.reset_color_pairs + +Discard all color-pair definitions. +[clinic start generated code]*/ + +static PyObject * +_curses_reset_color_pairs_impl(PyObject *module) +/*[clinic end generated code: output=117e68c6614e1d06 input=57c1cf7e5447e1ac]*/ +{ + PyCursesStatefulInitialised(module); + PyCursesStatefulInitialisedColor(module); + reset_color_pairs(); + Py_RETURN_NONE; +} +#endif /* _NCURSES_EXTENDED_COLOR_FUNCS */ + +/* Refresh the private copy of the screen encoding from a freshly created + stdscr window object. Returns 0 on success, -1 with an exception set. */ +static int +curses_update_screen_encoding(PyObject *winobj) +{ + char *copy = _PyMem_Strdup(((PyCursesWindowObject *)winobj)->encoding); + if (copy == NULL) { + PyErr_NoMemory(); + return -1; + } + PyMem_Free(curses_screen_encoding); + curses_screen_encoding = copy; + return 0; +} + +/* Populate the module dictionary with the ACS_* line-drawing constants and + LINES/COLS. These are only meaningful once a screen exists (after + initscr() or newterm()), which is why this is not done at module + initialisation. Returns 0 on success, -1 with an exception set. */ +static int +curses_init_dict(PyObject *module) +{ PyObject *module_dict = PyModule_GetDict(module); // borrowed if (module_dict == NULL) { - return NULL; + return -1; } /* This was moved from initcurses() because it core dumped on SGI, where they're not defined until you've called initscr() */ @@ -3758,12 +4743,12 @@ _curses_initscr_impl(PyObject *module) do { \ PyObject *value = PyLong_FromLong((long)(VALUE)); \ if (value == NULL) { \ - return NULL; \ + return -1; \ } \ int rc = PyDict_SetItemString(module_dict, (NAME), value); \ Py_DECREF(value); \ if (rc < 0) { \ - return NULL; \ + return -1; \ } \ } while (0) @@ -3837,13 +4822,63 @@ _curses_initscr_impl(PyObject *module) SetDictInt("LINES", LINES); SetDictInt("COLS", COLS); #undef SetDictInt + return 0; +} + +/*[clinic input] +_curses.initscr + +Initialize the library. + +Return a WindowObject which represents the whole screen. +[clinic start generated code]*/ + +static PyObject * +_curses_initscr_impl(PyObject *module) +/*[clinic end generated code: output=619fb68443810b7b input=514f4bce1821f6b5]*/ +{ + WINDOW *win; + + if (curses_initscr_called) { + cursesmodule_state *state = get_cursesmodule_state(module); + int code = wrefresh(stdscr); + if (code == ERR) { + _curses_set_null_error(state, "wrefresh", "initscr"); + return NULL; + } + PyObject *winobj = PyCursesWindow_New(state, stdscr, NULL, NULL, NULL); + if (winobj == NULL) { + return NULL; + } + if (curses_update_screen_encoding(winobj) < 0) { + Py_DECREF(winobj); + return NULL; + } + return winobj; + } + + win = initscr(); + + if (win == NULL) { + curses_set_null_error(module, "initscr", NULL); + return NULL; + } + + curses_initscr_called = curses_setupterm_called = TRUE; + + if (curses_init_dict(module) < 0) { + return NULL; + } cursesmodule_state *state = get_cursesmodule_state(module); - PyObject *winobj = PyCursesWindow_New(state, win, NULL, NULL); + PyObject *winobj = PyCursesWindow_New(state, win, NULL, NULL, NULL); if (winobj == NULL) { return NULL; } - curses_screen_encoding = ((PyCursesWindowObject *)winobj)->encoding; + if (curses_update_screen_encoding(winobj) < 0) { + Py_DECREF(winobj); + return NULL; + } return winobj; } @@ -3906,6 +4941,206 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) Py_RETURN_NONE; } +static int update_lines_cols(PyObject *private_module); /* defined below */ + +/* Return a file descriptor for obj, or, if obj is NULL or None, for the + sys.<stdname> stream. Returns -1 with an exception set on error. */ +static int +curses_fileno(PyObject *module, PyObject *obj, const char *stdname) +{ + if (obj != NULL && obj != Py_None) { + return PyObject_AsFileDescriptor(obj); + } + PyObject *stream; + if (PySys_GetOptionalAttrString(stdname, &stream) < 0) { + return -1; + } + if (stream == NULL || stream == Py_None) { + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_Format(state->error, "lost sys.%s", stdname); + Py_XDECREF(stream); + return -1; + } + int fd = PyObject_AsFileDescriptor(stream); + Py_DECREF(stream); + return fd; +} + +/* Duplicate fd and wrap it in a new (non-inheritable) stdio stream that the + screen object will own. Duplicating means closing the stream later does + not close the caller's fd. Returns NULL with an exception set on error. */ +static FILE * +curses_fdopen_dup(int fd, const char *mode) +{ + /* _Py_dup() duplicates the descriptor and makes the copy non-inheritable + atomically (and sets the error on failure). */ + int dfd = _Py_dup(fd); + if (dfd < 0) { + return NULL; + } + FILE *stream = fdopen(dfd, mode); + if (stream == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + close(dfd); + return NULL; + } + return stream; +} + +/*[clinic input] +_curses.newterm + + type: str(accept={str, NoneType}) = None + Terminal name; if None, the TERM environment variable is used. + fd: object = None + Output file object or descriptor (default: sys.stdout). + infd: object = None + Input file object or descriptor (default: sys.stdin). + / + +Return a new screen for the terminal, in addition to the initial screen. + +This is an alternative to initscr() for programs running on more than +one terminal. Use set_term() to switch between the screens. +[clinic start generated code]*/ + +static PyObject * +_curses_newterm_impl(PyObject *module, const char *type, PyObject *fd, + PyObject *infd) +/*[clinic end generated code: output=62663c31909d796c input=98507fe48c2e93cb]*/ +{ + /* Duplicate each descriptor right after resolving it: resolving the other + one runs arbitrary Python code (e.g. a fileno() method) that could close + this one before it is duplicated. */ + int out_fd = curses_fileno(module, fd, "stdout"); + if (out_fd < 0) { + return NULL; + } + FILE *outfp = curses_fdopen_dup(out_fd, "wb"); + if (outfp == NULL) { + return NULL; + } + + int in_fd = curses_fileno(module, infd, "stdin"); + if (in_fd < 0) { + fclose(outfp); + return NULL; + } + FILE *infp = curses_fdopen_dup(in_fd, "rb"); + if (infp == NULL) { + fclose(outfp); + return NULL; + } + + SCREEN *screen = newterm((char *)type, outfp, infp); + if (screen == NULL) { + curses_set_null_error(module, "newterm", NULL); + fclose(outfp); + fclose(infp); + return NULL; + } + /* newterm() makes the new screen the current one, so stdscr now refers + to its standard window. */ + curses_initscr_called = curses_setupterm_called = TRUE; + + cursesmodule_state *state = get_cursesmodule_state(module); + /* The screen object owns the SCREEN and the streams; deleting it (when it + is no longer referenced) calls delscreen() and closes the streams. */ + PyObject *screenobj = PyCursesScreen_New(state, screen, outfp, infp, NULL); + if (screenobj == NULL) { + delscreen(screen); + fclose(outfp); + fclose(infp); + return NULL; + } + /* The standard window keeps the screen alive for its own lifetime. */ + PyObject *win = PyCursesWindow_New(state, stdscr, NULL, NULL, screenobj); + if (win == NULL || + curses_update_screen_encoding(win) < 0 || + curses_init_dict(module) < 0) + { + Py_XDECREF(win); + Py_DECREF(screenobj); + return NULL; + } + ((PyCursesScreenObject *)screenobj)->stdscr = Py_NewRef(win); + Py_DECREF(win); + Py_XSETREF(state->topscreen, Py_NewRef(screenobj)); + return screenobj; +} + +/* Check that obj is an open screen object; returns it cast, or NULL with + TypeError/curses.error set. */ +static PyCursesScreenObject * +curses_check_screen(PyObject *module, PyObject *obj) +{ + cursesmodule_state *state = get_cursesmodule_state(module); + if (!PyObject_TypeCheck(obj, state->screen_type)) { + PyErr_Format(PyExc_TypeError, + "expected a curses screen, got %T", obj); + return NULL; + } + PyCursesScreenObject *so = _PyCursesScreenObject_CAST(obj); + if (so->screen == NULL) { + PyErr_SetString(state->error, "the screen has been deleted"); + return NULL; + } + return so; +} + +/*[clinic input] +_curses.set_term + + screen: object + / + +Switch to the given screen and return the previously current screen. + +Returns None if the previous screen was the one created by initscr(). +[clinic start generated code]*/ + +static PyObject * +_curses_set_term(PyObject *module, PyObject *screen) +/*[clinic end generated code: output=204cf9c40523bdef input=ed4dba18dd9adf6a]*/ +{ + PyCursesScreenObject *so = curses_check_screen(module, screen); + if (so == NULL) { + return NULL; + } + set_term(so->screen); + if (!update_lines_cols(module)) { + return NULL; + } + cursesmodule_state *state = get_cursesmodule_state(module); + PyObject *prev = state->topscreen; /* steal the owned reference */ + state->topscreen = Py_NewRef(screen); + return prev != NULL ? prev : Py_NewRef(Py_None); +} + +#ifdef HAVE_CURSES_NEW_PRESCR +/*[clinic input] +_curses.new_prescr + +Create a screen and return it, without initializing a terminal. + +The screen can be used to call functions that affect the screen before +calling newterm() or initscr(). +[clinic start generated code]*/ + +static PyObject * +_curses_new_prescr_impl(PyObject *module) +/*[clinic end generated code: output=e7de5031da7511e2 input=1a3a89d630b641c3]*/ +{ + SCREEN *screen = new_prescr(); + if (screen == NULL) { + curses_set_null_error(module, "new_prescr", NULL); + return NULL; + } + cursesmodule_state *state = get_cursesmodule_state(module); + return PyCursesScreen_New(state, screen, NULL, NULL, NULL); +} +#endif /* HAVE_CURSES_NEW_PRESCR */ + #if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102 // https://invisible-island.net/ncurses/NEWS.html#index-t20080119 @@ -3914,14 +5149,14 @@ _curses.get_escdelay Gets the curses ESCDELAY setting. -Gets the number of milliseconds to wait after reading an escape character, -to distinguish between an individual escape character entered on the -keyboard from escape sequences sent by cursor and function keys. +Gets the number of milliseconds to wait after reading an escape +character, to distinguish between an individual escape character entered +on the keyboard from escape sequences sent by cursor and function keys. [clinic start generated code]*/ static PyObject * _curses_get_escdelay_impl(PyObject *module) -/*[clinic end generated code: output=222fa1a822555d60 input=be2d5b3dd974d0a4]*/ +/*[clinic end generated code: output=222fa1a822555d60 input=b39eeae4b8f169ab]*/ { return PyLong_FromLong(ESCDELAY); } @@ -3933,14 +5168,14 @@ _curses.set_escdelay Sets the curses ESCDELAY setting. -Sets the number of milliseconds to wait after reading an escape character, -to distinguish between an individual escape character entered on the -keyboard from escape sequences sent by cursor and function keys. +Sets the number of milliseconds to wait after reading an escape +character, to distinguish between an individual escape character entered +on the keyboard from escape sequences sent by cursor and function keys. [clinic start generated code]*/ static PyObject * _curses_set_escdelay_impl(PyObject *module, int ms) -/*[clinic end generated code: output=43818efbf7980ac4 input=7796fe19f111e250]*/ +/*[clinic end generated code: output=43818efbf7980ac4 input=cc2529bcdda3b06c]*/ { if (ms <= 0) { PyErr_SetString(PyExc_ValueError, "ms must be > 0"); @@ -3955,13 +5190,13 @@ _curses.get_tabsize Gets the curses TABSIZE setting. -Gets the number of columns used by the curses library when converting a tab -character to spaces as it adds the tab to a window. +Gets the number of columns used by the curses library when converting +a tab character to spaces as it adds the tab to a window. [clinic start generated code]*/ static PyObject * _curses_get_tabsize_impl(PyObject *module) -/*[clinic end generated code: output=7e9e51fb6126fbdf input=74af86bf6c9f5d7e]*/ +/*[clinic end generated code: output=7e9e51fb6126fbdf input=58bdaacb337c103b]*/ { return PyLong_FromLong(TABSIZE); } @@ -3973,13 +5208,13 @@ _curses.set_tabsize Sets the curses TABSIZE setting. -Sets the number of columns used by the curses library when converting a tab -character to spaces as it adds the tab to a window. +Sets the number of columns used by the curses library when converting +a tab character to spaces as it adds the tab to a window. [clinic start generated code]*/ static PyObject * _curses_set_tabsize_impl(PyObject *module, int size) -/*[clinic end generated code: output=c1de5a76c0daab1e input=78cba6a3021ad061]*/ +/*[clinic end generated code: output=c1de5a76c0daab1e input=34c1be9a78cd28a2]*/ { if (size <= 0) { PyErr_SetString(PyExc_ValueError, "size must be > 0"); @@ -3996,11 +5231,16 @@ _curses.intrflush flag: bool / +Control flushing of the output buffer when an interrupt key is pressed. + +If flag is true, pressing an interrupt key (interrupt, break, or quit) +flushes all output in the terminal driver queue. If flag is false, no +flushing is done. [clinic start generated code]*/ static PyObject * _curses_intrflush_impl(PyObject *module, int flag) -/*[clinic end generated code: output=c1986df35e999a0f input=c65fe2ef973fe40a]*/ +/*[clinic end generated code: output=c1986df35e999a0f input=66588c2bccc7e8fa]*/ { PyCursesStatefulInitialised(module); @@ -4086,19 +5326,39 @@ _curses_killchar_impl(PyObject *module) return PyBytes_FromStringAndSize(&ch, 1); } +#ifdef HAVE_NCURSESW +/*[clinic input] +_curses.killwchar + +Return the user's current wide-character line kill character. +[clinic start generated code]*/ + +static PyObject * +_curses_killwchar_impl(PyObject *module) +/*[clinic end generated code: output=eac1fd72a0c88d42 input=5c2d7d1ab2f24eb7]*/ +{ + wchar_t ch; + + if (killwchar(&ch) == ERR) { + curses_set_error(module, "killwchar", NULL); + return NULL; + } + return PyUnicode_FromWideChar(&ch, 1); +} +#endif /* HAVE_NCURSESW */ + /*[clinic input] -@permit_long_docstring_body _curses.longname Return the terminfo long name field describing the current terminal. -The maximum length of a verbose description is 128 characters. It is defined -only after the call to initscr(). +The maximum length of a verbose description is 128 characters. It is +defined only after the call to initscr(). [clinic start generated code]*/ static PyObject * _curses_longname_impl(PyObject *module) -/*[clinic end generated code: output=fdf30433727ef568 input=5de06852f2230ddb]*/ +/*[clinic end generated code: output=fdf30433727ef568 input=a924fabba0de78a6]*/ NoArgReturnStringFunctionBody(longname) /*[clinic input] @@ -4133,13 +5393,13 @@ _curses.mouseinterval Set and retrieve the maximum time between press and release in a click. Set the maximum time that can elapse between press and release events in -order for them to be recognized as a click, and return the previous interval -value. +order for them to be recognized as a click, and return the previous +interval value. [clinic start generated code]*/ static PyObject * _curses_mouseinterval_impl(PyObject *module, int interval) -/*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ +/*[clinic end generated code: output=c4f5ff04354634c5 input=b90249254389c080]*/ { PyCursesStatefulInitialised(module); int value = mouseinterval(interval); @@ -4151,23 +5411,22 @@ _curses_mouseinterval_impl(PyObject *module, int interval) } /*[clinic input] -@permit_long_summary _curses.mousemask newmask: unsigned_long(bitwise=True) / -Set the mouse events to be reported, and return a tuple (availmask, oldmask). +Set the mouse events to be reported, and return (availmask, oldmask). Return a tuple (availmask, oldmask). availmask indicates which of the -specified mouse events can be reported; on complete failure it returns 0. -oldmask is the previous value of the given window's mouse event mask. -If this function is never called, no mouse events are ever reported. +specified mouse events can be reported; on complete failure it returns +0. oldmask is the previous value of the mouse event mask. If this +function is never called, no mouse events are ever reported. [clinic start generated code]*/ static PyObject * _curses_mousemask_impl(PyObject *module, unsigned long newmask) -/*[clinic end generated code: output=9406cf1b8a36e485 input=b92ff4fbe5ce61b1]*/ +/*[clinic end generated code: output=9406cf1b8a36e485 input=b8a9a4ccbce633f4]*/ { mmask_t oldmask, availmask; @@ -4229,7 +5488,7 @@ _curses_newpad_impl(PyObject *module, int nlines, int ncols) } cursesmodule_state *state = get_cursesmodule_state(module); - return PyCursesWindow_New(state, win, NULL, NULL); + return PyCursesWindow_New(state, win, NULL, NULL, state->topscreen); } /*[clinic input] @@ -4249,14 +5508,14 @@ _curses.newwin Return a new window. -By default, the window will extend from the specified position to the lower -right corner of the screen. +By default, the window will extend from the specified position to the +lower right corner of the screen. [clinic start generated code]*/ static PyObject * _curses_newwin_impl(PyObject *module, int nlines, int ncols, int group_right_1, int begin_y, int begin_x) -/*[clinic end generated code: output=c1e0a8dc8ac2826c input=29312c15a72a003d]*/ +/*[clinic end generated code: output=c1e0a8dc8ac2826c input=a1517cbfea4ab24b]*/ { WINDOW *win; @@ -4269,7 +5528,7 @@ _curses_newwin_impl(PyObject *module, int nlines, int ncols, } cursesmodule_state *state = get_cursesmodule_state(module); - return PyCursesWindow_New(state, win, NULL, NULL); + return PyCursesWindow_New(state, win, NULL, NULL, state->topscreen); } /*[clinic input] @@ -4281,13 +5540,14 @@ _curses.nl Enter newline mode. -This mode translates the return key into newline on input, and translates -newline into return and line-feed on output. Newline mode is initially on. +This mode translates the return key into newline on input, and +translates newline into return and line-feed on output. Newline mode +is initially on. [clinic start generated code]*/ static PyObject * _curses_nl_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b39cc0ffc9015003 input=18e3e9c6e8cfcf6f]*/ +/*[clinic end generated code: output=b39cc0ffc9015003 input=3fb21dcf55521ee4]*/ NoArgOrFlagNoReturnFunctionBody(nl, flag) /*[clinic input] @@ -4321,13 +5581,13 @@ _curses.nonl Leave newline mode. -Disable translation of return into newline on input, and disable low-level -translation of newline into newline/return on output. +Disable translation of return into newline on input, and disable +low-level translation of newline into newline/return on output. [clinic start generated code]*/ static PyObject * _curses_nonl_impl(PyObject *module) -/*[clinic end generated code: output=99e917e9715770c6 input=9d37dd122d3022fc]*/ +/*[clinic end generated code: output=99e917e9715770c6 input=75cce08e4b6b3ef1]*/ NoArgNoReturnFunctionBody(nonl) /*[clinic input] @@ -4358,6 +5618,7 @@ _curses_noraw_impl(PyObject *module) NoArgNoReturnFunctionBody(noraw) /*[clinic input] +@permit_long_summary _curses.pair_content pair_number: pair @@ -4369,7 +5630,7 @@ Return a tuple (fg, bg) containing the colors for the requested color pair. static PyObject * _curses_pair_content_impl(PyObject *module, int pair_number) -/*[clinic end generated code: output=4a726dd0e6885f3f input=03970f840fc7b739]*/ +/*[clinic end generated code: output=4a726dd0e6885f3f input=faede9e26f1f2ca4]*/ { _CURSES_COLOR_NUM_TYPE f, b; @@ -4393,6 +5654,7 @@ _curses_pair_content_impl(PyObject *module, int pair_number) } /*[clinic input] +@permit_long_summary _curses.pair_number attr: int @@ -4405,7 +5667,7 @@ color_pair() is the counterpart to this function. static PyObject * _curses_pair_number_impl(PyObject *module, int attr) -/*[clinic end generated code: output=85bce7d65c0aa3f4 input=d478548e33f5e61a]*/ +/*[clinic end generated code: output=85bce7d65c0aa3f4 input=b11152a78c2f9abf]*/ { PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); @@ -4414,6 +5676,7 @@ _curses_pair_number_impl(PyObject *module, int attr) } /*[clinic input] +@permit_long_summary _curses.putp string: str(accept={robuffer}) @@ -4426,7 +5689,7 @@ Note that the output of putp() always goes to standard output. static PyObject * _curses_putp_impl(PyObject *module, const char *string) -/*[clinic end generated code: output=e98081d1b8eb5816 input=1601faa828b44cb3]*/ +/*[clinic end generated code: output=e98081d1b8eb5816 input=2f3b9e0f22829ee7]*/ { return curses_check_err(module, putp(string), "putp", NULL); } @@ -4515,11 +5778,14 @@ update_lines_cols(PyObject *private_module) /*[clinic input] _curses.update_lines_cols +Update the LINES and COLS module variables. + +This is useful for detecting manual screen resize. [clinic start generated code]*/ static PyObject * _curses_update_lines_cols_impl(PyObject *module) -/*[clinic end generated code: output=423f2b1e63ed0f75 input=5f065ab7a28a5d90]*/ +/*[clinic end generated code: output=423f2b1e63ed0f75 input=1d8ea7c356b61a8b]*/ { if (!update_lines_cols(module)) { return NULL; @@ -4539,13 +5805,13 @@ _curses.raw Enter raw mode. In raw mode, normal line buffering and processing of interrupt, quit, -suspend, and flow control keys are turned off; characters are presented to -curses input functions one by one. +suspend, and flow control keys are turned off; characters are presented +to curses input functions one by one. [clinic start generated code]*/ static PyObject * _curses_raw_impl(PyObject *module, int flag) -/*[clinic end generated code: output=a750e4b342be015b input=4b447701389fb4df]*/ +/*[clinic end generated code: output=a750e4b342be015b input=18a7de7eef16987a]*/ NoArgOrFlagNoReturnFunctionBody(raw, flag) /*[clinic input] @@ -4595,13 +5861,13 @@ _curses.resizeterm Resize the standard and current windows to the specified dimensions. -Adjusts other bookkeeping data used by the curses library that record the -window dimensions (in particular the SIGWINCH handler). +Adjusts other bookkeeping data used by the curses library that record +the window dimensions (in particular the SIGWINCH handler). [clinic start generated code]*/ static PyObject * _curses_resizeterm_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/ +/*[clinic end generated code: output=4de3abab50c67f02 input=7f0f077df2da1cf5]*/ { PyObject *result; int code; @@ -4623,7 +5889,6 @@ _curses_resizeterm_impl(PyObject *module, short nlines, short ncols) #ifdef HAVE_CURSES_RESIZE_TERM /*[clinic input] -@permit_long_docstring_body _curses.resize_term nlines: short @@ -4635,15 +5900,16 @@ _curses.resize_term Backend function used by resizeterm(), performing most of the work. When resizing the windows, resize_term() blank-fills the areas that are -extended. The calling application should fill in these areas with appropriate -data. The resize_term() function attempts to resize all windows. However, -due to the calling convention of pads, it is not possible to resize these -without additional interaction with the application. +extended. The calling application should fill in these areas with +appropriate data. The resize_term() function attempts to resize all +windows. However, due to the calling convention of pads, it is not +possible to resize these without additional interaction with the +application. [clinic start generated code]*/ static PyObject * _curses_resize_term_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=46c6d749fa291dbd input=ebfa840f6b5f03fa]*/ +/*[clinic end generated code: output=46c6d749fa291dbd input=ff4baaf2320c8ac9]*/ { PyObject *result; int code; @@ -4701,21 +5967,22 @@ _curses_setsyx_impl(PyObject *module, int y, int x) #endif /*[clinic input] +@permit_long_summary _curses.start_color Initializes eight basic colors and global variables COLORS and COLOR_PAIRS. -Must be called if the programmer wants to use colors, and before any other -color manipulation routine is called. It is good practice to call this -routine right after initscr(). +Must be called if the programmer wants to use colors, and before any +other color manipulation routine is called. It is good practice to call +this routine right after initscr(). -It also restores the colors on the terminal to the values they had when the -terminal was just turned on. +It also restores the colors on the terminal to the values they had when +the terminal was just turned on. [clinic start generated code]*/ static PyObject * _curses_start_color_impl(PyObject *module) -/*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ +/*[clinic end generated code: output=8b772b41d8090ede input=7daacc6b6baba643]*/ { PyCursesStatefulInitialised(module); @@ -4804,13 +6071,13 @@ _curses.tigetnum Return the value of the numeric capability. -The value -2 is returned if capname is not a numeric capability, or -1 if -it is canceled or absent from the terminal description. +The value -2 is returned if capname is not a numeric capability, or -1 +if it is canceled or absent from the terminal description. [clinic start generated code]*/ static PyObject * _curses_tigetnum_impl(PyObject *module, const char *capname) -/*[clinic end generated code: output=46f8b0a1b5dff42f input=5cdf2f410b109720]*/ +/*[clinic end generated code: output=46f8b0a1b5dff42f input=87a64beec16ae077]*/ { PyCursesStatefulSetupTermCalled(module); @@ -4826,13 +6093,13 @@ _curses.tigetstr Return the value of the string capability. -None is returned if capname is not a string capability, or is canceled or -absent from the terminal description. +None is returned if capname is not a string capability, or is canceled +or absent from the terminal description. [clinic start generated code]*/ static PyObject * _curses_tigetstr_impl(PyObject *module, const char *capname) -/*[clinic end generated code: output=f22b576ad60248f3 input=36644df25c73c0a7]*/ +/*[clinic end generated code: output=f22b576ad60248f3 input=00bf0feda2207724]*/ { PyCursesStatefulSetupTermCalled(module); @@ -4934,6 +6201,52 @@ _curses_unctrl(PyObject *module, PyObject *ch) return PyBytes_FromString(res); } +#ifdef HAVE_NCURSESW +/*[clinic input] +_curses.wunctrl + + ch: object + / + +Return a printable representation of the wide character ch. + +Control characters are displayed as a caret followed by the character, +for example as ^C. Printing characters are left as they are. +[clinic start generated code]*/ + +static PyObject * +_curses_wunctrl(PyObject *module, PyObject *ch) +/*[clinic end generated code: output=7b16d5534ff05728 input=9ceb6749118bd07c]*/ +{ + chtype ch_; + wchar_t wstr[CCHARW_MAX + 1]; + cchar_t wcval; + + PyCursesStatefulInitialised(module); + + int type = PyCurses_ConvertToCchar_t(NULL, ch, &ch_, wstr); + if (type == 0) { + return NULL; + } + if (type == 1) { + /* A narrow character is the spacing character of the cell. */ + wstr[0] = (wchar_t)(ch_ & A_CHARTEXT); + wstr[1] = L'\0'; + } + if (setcchar(&wcval, wstr, A_NORMAL, 0, NULL) == ERR) { + curses_set_error(module, "setcchar", "wunctrl"); + return NULL; + } + + wchar_t *res = wunctrl(&wcval); + if (res == NULL) { + curses_set_null_error(module, "wunctrl", NULL); + return NULL; + } + return PyUnicode_FromWideChar(res, -1); +} +#endif /* HAVE_NCURSESW */ + /*[clinic input] _curses.ungetch @@ -5030,7 +6343,6 @@ _curses_unget_wch(PyObject *module, PyObject *ch) #ifdef HAVE_CURSES_USE_ENV /*[clinic input] -@permit_long_docstring_body _curses.use_env flag: bool @@ -5038,19 +6350,19 @@ _curses.use_env Use environment variables LINES and COLUMNS. -If used, this function should be called before initscr() or newterm() are -called. +If used, this function should be called before initscr() or newterm() +are called. -When flag is False, the values of lines and columns specified in the terminfo -database will be used, even if environment variables LINES and COLUMNS (used -by default) are set, or if curses is running in a window (in which case -default behavior would be to use the window size if LINES and COLUMNS are -not set). +When flag is False, the values of lines and columns specified in the +terminfo database will be used, even if environment variables LINES and +COLUMNS (used by default) are set, or if curses is running in a window +(in which case default behavior would be to use the window size if LINES +and COLUMNS are not set). [clinic start generated code]*/ static PyObject * _curses_use_env_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b2c445e435c0b164 input=eaa9047ec73c27d3]*/ +/*[clinic end generated code: output=b2c445e435c0b164 input=8e8feed746cf7fc1]*/ { use_env(flag); Py_RETURN_NONE; @@ -5078,6 +6390,7 @@ _curses_use_default_colors_impl(PyObject *module) } /*[clinic input] +@permit_long_summary _curses.assume_default_colors fg: int bg: int @@ -5093,7 +6406,7 @@ Use this to support transparency in your application. static PyObject * _curses_assume_default_colors_impl(PyObject *module, int fg, int bg) -/*[clinic end generated code: output=54985397a7d2b3a5 input=7fe301712ef3e9fb]*/ +/*[clinic end generated code: output=54985397a7d2b3a5 input=8945333c09893cf2]*/ { int code; @@ -5162,6 +6475,7 @@ make_ncurses_version(PyTypeObject *type) #endif /* NCURSES_VERSION */ /*[clinic input] +@permit_long_summary _curses.has_extended_color_support Return True if the module supports extended colors; otherwise, return False. @@ -5172,7 +6486,7 @@ that support more than 16 colors (e.g. xterm-256color). static PyObject * _curses_has_extended_color_support_impl(PyObject *module) -/*[clinic end generated code: output=68f1be2b57d92e22 input=4b905f046e35ee9f]*/ +/*[clinic end generated code: output=68f1be2b57d92e22 input=40d673471c5056f0]*/ { return PyBool_FromLong(_NCURSES_EXTENDED_COLOR_FUNCS); } @@ -5180,6 +6494,7 @@ _curses_has_extended_color_support_impl(PyObject *module) /* List of functions defined in the module */ static PyMethodDef cursesmodule_methods[] = { + _CURSES_ALLOC_PAIR_METHODDEF _CURSES_BAUDRATE_METHODDEF _CURSES_BEEP_METHODDEF _CURSES_CAN_CHANGE_COLOR_METHODDEF @@ -5194,9 +6509,13 @@ static PyMethodDef cursesmodule_methods[] = { _CURSES_ECHO_METHODDEF _CURSES_ENDWIN_METHODDEF _CURSES_ERASECHAR_METHODDEF + _CURSES_ERASEWCHAR_METHODDEF _CURSES_FILTER_METHODDEF + _CURSES_NOFILTER_METHODDEF + _CURSES_FIND_PAIR_METHODDEF _CURSES_FLASH_METHODDEF _CURSES_FLUSHINP_METHODDEF + _CURSES_FREE_PAIR_METHODDEF _CURSES_GETMOUSE_METHODDEF _CURSES_UNGETMOUSE_METHODDEF _CURSES_GETSYX_METHODDEF @@ -5211,17 +6530,26 @@ static PyMethodDef cursesmodule_methods[] = { _CURSES_INIT_PAIR_METHODDEF _CURSES_INITSCR_METHODDEF _CURSES_INTRFLUSH_METHODDEF + _CURSES_IS_CBREAK_METHODDEF + _CURSES_IS_ECHO_METHODDEF + _CURSES_IS_NL_METHODDEF + _CURSES_IS_RAW_METHODDEF _CURSES_ISENDWIN_METHODDEF _CURSES_IS_TERM_RESIZED_METHODDEF _CURSES_KEYNAME_METHODDEF _CURSES_KILLCHAR_METHODDEF + _CURSES_KILLWCHAR_METHODDEF _CURSES_LONGNAME_METHODDEF _CURSES_META_METHODDEF _CURSES_MOUSEINTERVAL_METHODDEF _CURSES_MOUSEMASK_METHODDEF _CURSES_NAPMS_METHODDEF _CURSES_NEWPAD_METHODDEF + _CURSES_NEWTERM_METHODDEF _CURSES_NEWWIN_METHODDEF +#ifdef HAVE_CURSES_NEW_PRESCR + _CURSES_NEW_PRESCR_METHODDEF +#endif _CURSES_NL_METHODDEF _CURSES_NOCBREAK_METHODDEF _CURSES_NOECHO_METHODDEF @@ -5233,6 +6561,7 @@ static PyMethodDef cursesmodule_methods[] = { _CURSES_PUTP_METHODDEF _CURSES_QIFLUSH_METHODDEF _CURSES_RAW_METHODDEF + _CURSES_RESET_COLOR_PAIRS_METHODDEF _CURSES_RESET_PROG_MODE_METHODDEF _CURSES_RESET_SHELL_MODE_METHODDEF _CURSES_RESETTY_METHODDEF @@ -5245,6 +6574,7 @@ static PyMethodDef cursesmodule_methods[] = { #endif _CURSES_GET_TABSIZE_METHODDEF _CURSES_SET_TABSIZE_METHODDEF + _CURSES_SET_TERM_METHODDEF _CURSES_SETSYX_METHODDEF _CURSES_SETUPTERM_METHODDEF _CURSES_START_COLOR_METHODDEF @@ -5256,6 +6586,7 @@ static PyMethodDef cursesmodule_methods[] = { _CURSES_TPARM_METHODDEF _CURSES_TYPEAHEAD_METHODDEF _CURSES_UNCTRL_METHODDEF + _CURSES_WUNCTRL_METHODDEF _CURSES_UNGETCH_METHODDEF _CURSES_UPDATE_LINES_COLS_METHODDEF _CURSES_UNGET_WCH_METHODDEF @@ -5368,6 +6699,8 @@ cursesmodule_traverse(PyObject *mod, visitproc visit, void *arg) cursesmodule_state *state = get_cursesmodule_state(mod); Py_VISIT(state->error); Py_VISIT(state->window_type); + Py_VISIT(state->screen_type); + Py_VISIT(state->topscreen); return 0; } @@ -5377,6 +6710,8 @@ cursesmodule_clear(PyObject *mod) cursesmodule_state *state = get_cursesmodule_state(mod); Py_CLEAR(state->error); Py_CLEAR(state->window_type); + Py_CLEAR(state->screen_type); + Py_CLEAR(state->topscreen); return 0; } @@ -5384,6 +6719,8 @@ static void cursesmodule_free(void *mod) { (void)cursesmodule_clear((PyObject *)mod); + PyMem_Free(curses_screen_encoding); + curses_screen_encoding = NULL; curses_module_loaded = 0; // allow reloading once garbage-collected } @@ -5407,6 +6744,14 @@ cursesmodule_exec(PyObject *module) if (PyModule_AddType(module, state->window_type) < 0) { return -1; } + state->screen_type = (PyTypeObject *)PyType_FromModuleAndSpec( + module, &PyCursesScreen_Type_spec, NULL); + if (state->screen_type == NULL) { + return -1; + } + if (PyModule_AddType(module, state->screen_type) < 0) { + return -1; + } /* Add some symbolic constants to the module */ PyObject *module_dict = PyModule_GetDict(module); @@ -5432,7 +6777,10 @@ cursesmodule_exec(PyObject *module) } /* For exception curses.error */ - state->error = PyErr_NewException("_curses.error", NULL, NULL); + state->error = PyErr_NewExceptionWithDoc( + "_curses.error", + "Exception raised when a curses library function returns an error.", + NULL, NULL); if (state->error == NULL) { return -1; } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 163e499d957b2e2..fd8d95d05c933e0 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -126,8 +126,8 @@ get_module_state(PyObject *module) #define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) -static PyObject * -get_current_module(PyInterpreterState *interp) +static int +get_current_module(PyInterpreterState *interp, PyObject **p_mod) { PyObject *mod = NULL; @@ -139,20 +139,24 @@ get_current_module(PyInterpreterState *interp) if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - if (ref != NULL) { - if (ref != Py_None) { - (void)PyWeakref_GetRef(ref, &mod); - if (mod == Py_None) { - Py_CLEAR(mod); - } + if (ref != NULL && ref != Py_None) { + if (PyWeakref_GetRef(ref, &mod) < 0) { Py_DECREF(ref); + goto error; + } + if (mod == Py_None) { + Py_CLEAR(mod); } + Py_DECREF(ref); } - return mod; + assert(!PyErr_Occurred()); + *p_mod = mod; + return mod != NULL; error: assert(PyErr_Occurred()); - return NULL; + *p_mod = NULL; + return -1; } static PyModuleDef datetimemodule; @@ -161,22 +165,26 @@ static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *mod = get_current_module(interp); + PyObject *mod; + if (get_current_module(interp, &mod) < 0) { + goto error; + } if (mod == NULL) { - assert(!PyErr_Occurred()); - if (PyErr_Occurred()) { - return NULL; - } /* The static types can outlive the module, * so we must re-import the module. */ mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { - return NULL; + goto error; } } datetime_state *st = get_module_state(mod); *p_mod = mod; return st; + +error: + assert(PyErr_Occurred()); + *p_mod = NULL; + return NULL; } #define GET_CURRENT_STATE(MOD_VAR) \ @@ -1509,7 +1517,7 @@ get_tzinfo_member(PyObject *self) * this returns NULL. Else result is returned. */ static PyObject * -call_tzinfo_method(PyObject *tzinfo, const char *name, PyObject *tzinfoarg) +call_tzinfo_method(PyObject *tzinfo, PyObject *name, PyObject *tzinfoarg) { PyObject *offset; @@ -1519,7 +1527,7 @@ call_tzinfo_method(PyObject *tzinfo, const char *name, PyObject *tzinfoarg) if (tzinfo == Py_None) Py_RETURN_NONE; - offset = PyObject_CallMethod(tzinfo, name, "O", tzinfoarg); + offset = PyObject_CallMethodOneArg(tzinfo, name, tzinfoarg); if (offset == Py_None || offset == NULL) return offset; if (PyDelta_Check(offset)) { @@ -1536,7 +1544,7 @@ call_tzinfo_method(PyObject *tzinfo, const char *name, PyObject *tzinfoarg) } else { PyErr_Format(PyExc_TypeError, - "tzinfo.%s() must return None or " + "tzinfo.%U() must return None or " "timedelta, not '%.200s'", name, Py_TYPE(offset)->tp_name); Py_DECREF(offset); @@ -1557,7 +1565,7 @@ call_tzinfo_method(PyObject *tzinfo, const char *name, PyObject *tzinfoarg) static PyObject * call_utcoffset(PyObject *tzinfo, PyObject *tzinfoarg) { - return call_tzinfo_method(tzinfo, "utcoffset", tzinfoarg); + return call_tzinfo_method(tzinfo, &_Py_ID(utcoffset), tzinfoarg); } /* Call tzinfo.dst(tzinfoarg), and extract an integer from the @@ -1571,7 +1579,7 @@ call_utcoffset(PyObject *tzinfo, PyObject *tzinfoarg) static PyObject * call_dst(PyObject *tzinfo, PyObject *tzinfoarg) { - return call_tzinfo_method(tzinfo, "dst", tzinfoarg); + return call_tzinfo_method(tzinfo, &_Py_ID(dst), tzinfoarg); } /* Call tzinfo.tzname(tzinfoarg), and return the result. tzinfo must be @@ -1660,8 +1668,8 @@ tzinfo_from_isoformat_results(int rv, int tzoffset, int tz_useconds) { PyObject *tzinfo; if (rv == 1) { - // Create a timezone from offset in seconds (0 returns UTC) - if (tzoffset == 0) { + // Create a timezone from the offset (a zero offset returns UTC) + if (tzoffset == 0 && tz_useconds == 0) { return Py_NewRef(CONST_UTC(NO_STATE)); } @@ -2128,8 +2136,11 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) @@ -2207,8 +2218,11 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { @@ -2815,8 +2829,11 @@ delta_new_impl(PyTypeObject *type, PyObject *days, PyObject *seconds, { PyObject *self = NULL; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } PyObject *x = NULL; /* running sum of microseconds */ PyObject *y = NULL; /* temp sum of microseconds */ @@ -3014,8 +3031,12 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy)) if (total_microseconds == NULL) return NULL; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + Py_DECREF(total_microseconds); + return NULL; + } total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); @@ -3325,7 +3346,6 @@ datetime_date_today_impl(PyTypeObject *type) } /*[clinic input] -@permit_long_docstring_body @classmethod datetime.date.fromtimestamp @@ -3334,13 +3354,13 @@ datetime.date.fromtimestamp Create a date from a POSIX timestamp. -The timestamp is a number, e.g. created via time.time(), that is interpreted -as local time. +The timestamp is a number, e.g. created via time.time(), that is +interpreted as local time. [clinic start generated code]*/ static PyObject * datetime_date_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp) -/*[clinic end generated code: output=59def4e32c028fb6 input=55ff6940f0a8339f]*/ +/*[clinic end generated code: output=59def4e32c028fb6 input=15720eef43b169a1]*/ { return date_fromtimestamp(type, timestamp); } @@ -3476,6 +3496,7 @@ datetime_date_fromisocalendar_impl(PyTypeObject *type, int year, int week, } /*[clinic input] +@permit_long_summary @classmethod datetime.date.strptime @@ -3492,7 +3513,7 @@ For a list of supported format codes, see the documentation: static PyObject * datetime_date_strptime_impl(PyTypeObject *type, PyObject *string, PyObject *format) -/*[clinic end generated code: output=454d473bee2d5161 input=31d57bb789433e99]*/ +/*[clinic end generated code: output=454d473bee2d5161 input=2db8f0b2b5242deb]*/ { PyObject *result; @@ -3867,8 +3888,11 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy)) week = 0; } - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); @@ -4744,6 +4768,7 @@ datetime_time_impl(PyTypeObject *type, int hour, int minute, int second, } /*[clinic input] +@permit_long_summary @classmethod datetime.time.strptime @@ -4760,7 +4785,7 @@ For a list of supported format codes, see the documentation: static PyObject * datetime_time_strptime_impl(PyTypeObject *type, PyObject *string, PyObject *format) -/*[clinic end generated code: output=ae05a9bc0241d3bf input=82ba425ecacc54aa]*/ +/*[clinic end generated code: output=ae05a9bc0241d3bf input=f01d0b9eb5383da5]*/ { PyObject *result; @@ -4856,8 +4881,8 @@ datetime.time.isoformat Return the time formatted according to ISO. -The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the fractional -part is omitted if self.microsecond == 0. +The full format is 'HH:MM:SS.mmmmmm+zz:zz'. By default, the +fractional part is omitted if self.microsecond == 0. The optional argument timespec specifies the number of additional terms of the time to include. Valid options are 'auto', 'hours', @@ -4866,7 +4891,7 @@ terms of the time to include. Valid options are 'auto', 'hours', static PyObject * datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec) -/*[clinic end generated code: output=2bcc7cab65c35545 input=afbbbd953d10ad07]*/ +/*[clinic end generated code: output=2bcc7cab65c35545 input=0efae103081060f4]*/ { char buf[100]; @@ -4927,14 +4952,14 @@ datetime_time_isoformat_impl(PyDateTime_Time *self, const char *timespec) } /*[clinic input] -@permit_long_docstring_body datetime.time.strftime format: unicode Format using strftime(). -The date part of the timestamp passed to underlying strftime should not be used. +The date part of the timestamp passed to underlying strftime should +not be used. For a list of supported format codes, see the documentation: https://docs.python.org/3/library/datetime.html#format-codes @@ -4942,7 +4967,7 @@ For a list of supported format codes, see the documentation: static PyObject * datetime_time_strftime_impl(PyDateTime_Time *self, PyObject *format) -/*[clinic end generated code: output=10f65af20e2a78c7 input=c4a5bbecd798654b]*/ +/*[clinic end generated code: output=10f65af20e2a78c7 input=184e1c0d7d356c5d]*/ { PyObject *result; PyObject *tuple; @@ -5510,15 +5535,15 @@ datetime.datetime.__new__ A combination of a date and a time. -The year, month and day arguments are required. tzinfo may be None, or an -instance of a tzinfo subclass. The remaining arguments may be ints. +The year, month and day arguments are required. tzinfo may be None, or +an instance of a tzinfo subclass. The remaining arguments may be ints. [clinic start generated code]*/ static PyObject * datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, int hour, int minute, int second, int microsecond, PyObject *tzinfo, int fold) -/*[clinic end generated code: output=47983ddb47d36037 input=2af468d7a9c1e568]*/ +/*[clinic end generated code: output=47983ddb47d36037 input=c7fd85dcf6fe9691]*/ { return new_datetime_ex2(year, month, day, hour, minute, second, microsecond, @@ -5735,7 +5760,6 @@ datetime_datetime_utcnow_impl(PyTypeObject *type) } /*[clinic input] -@permit_long_docstring_body @classmethod datetime.datetime.fromtimestamp @@ -5744,14 +5768,14 @@ datetime.datetime.fromtimestamp Create a datetime from a POSIX timestamp. -The timestamp is a number, e.g. created via time.time(), that is interpreted -as local time. +The timestamp is a number, e.g. created via time.time(), that is +interpreted as local time. [clinic start generated code]*/ static PyObject * datetime_datetime_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp, PyObject *tzinfo) -/*[clinic end generated code: output=9c47ea2b2ebdaded input=d6b5b2095c5a34b2]*/ +/*[clinic end generated code: output=9c47ea2b2ebdaded input=7a2bc81a049ea287]*/ { PyObject *self; if (check_tzinfo_subclass(tzinfo) < 0) @@ -6120,7 +6144,7 @@ datetime_datetime_fromisoformat_impl(PyTypeObject *type, PyObject *string) goto error; } - if ((hour == 24) && (month <= 12)) { + if ((hour == 24) && (month >= 1 && month <= 12)) { int d_in_month = days_in_month(year, month); if (day <= d_in_month) { if (minute == 0 && second == 0 && microsecond == 0) { @@ -6800,8 +6824,11 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); RELEASE_CURRENT_STATE(st, current_mod); @@ -7047,8 +7074,11 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } PyObject *delta; delta = datetime_subtract(op, CONST_EPOCH(st)); @@ -7581,9 +7611,8 @@ _datetime_exec(PyObject *module) datetime_state *st = get_module_state(module); PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *old_module = get_current_module(interp); - if (PyErr_Occurred()) { - assert(old_module == NULL); + PyObject *old_module; + if (get_current_module(interp, &old_module) < 0) { goto error; } /* We actually set the "current" module right before a successful return. */ diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 6b07ef74cfa51db..a9f4f27d9eb742e 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -431,13 +431,14 @@ _dbm.dbm.setdefault Return the value for key if present, otherwise default. -If key is not in the database, it is inserted with default as the value. +If key is not in the database, it is inserted with default as the +value. [clinic start generated code]*/ static PyObject * _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value) -/*[clinic end generated code: output=9c2f6ea6d0fb576c input=c01510ef7571e13b]*/ +/*[clinic end generated code: output=9c2f6ea6d0fb576c input=81224965c110f830]*/ { datum dbm_key, val; Py_ssize_t tmp_size; diff --git a/Modules/_decimal/README.txt b/Modules/_decimal/README.txt index 7eae0f88f284280..0fa161c8c845553 100644 --- a/Modules/_decimal/README.txt +++ b/Modules/_decimal/README.txt @@ -1,5 +1,3 @@ - - About ===== @@ -7,40 +5,3 @@ _decimal.c is a wrapper for the libmpdec library. libmpdec is a fast C library for correctly-rounded arbitrary precision decimal floating point arithmetic. It is a complete implementation of Mike Cowlishaw/IBM's General Decimal Arithmetic Specification. - - -Build process for the module -============================ - -As usual, the build process for _decimal.so is driven by setup.py in the top -level directory. setup.py autodetects the following build configurations: - - 1) x64 - 64-bit Python, x86_64 processor (AMD, Intel) - - 2) uint128 - 64-bit Python, compiler provides __uint128_t (gcc) - - 3) ansi64 - 64-bit Python, ANSI C - - 4) ppro - 32-bit Python, x86 CPU, PentiumPro or later - - 5) ansi32 - 32-bit Python, ANSI C - - 6) ansi-legacy - 32-bit Python, compiler without uint64_t - - 7) universal - Mac OS only (multi-arch) - - -It is possible to override autodetection by exporting: - - PYTHON_DECIMAL_WITH_MACHINE=value, where value is one of the above options. - - -NOTE -==== - -decimal.so is not built from a static libmpdec.a since doing so led to -failures on AIX (user report) and Windows (mixing static and dynamic CRTs -causes locale problems and more). - - - diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 0a8308d9ebce7a7..4db1b60be77844d 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -188,7 +188,7 @@ find_state_ternary(PyObject *left, PyObject *right, PyObject *modulus) * sizeof(size_t) == sizeof(mpd_uint_t) == sizeof(mpd_ssize_t) */ -#ifdef TEST_COVERAGE +#ifdef Py_DEBUG #undef Py_LOCAL_INLINE #define Py_LOCAL_INLINE Py_LOCAL #endif @@ -939,13 +939,13 @@ _decimal.Context.Etop Return a value equal to Emax - prec + 1. -This is the maximum exponent if the _clamp field of the context is set -to 1 (IEEE clamp mode). Etop() must not be negative. +This is the maximum exponent if the _clamp field of the context is +set to 1 (IEEE clamp mode). Etop() must not be negative. [clinic start generated code]*/ static PyObject * _decimal_Context_Etop_impl(PyObject *self) -/*[clinic end generated code: output=f0a3f6e1b829074e input=838a4409316ec728]*/ +/*[clinic end generated code: output=f0a3f6e1b829074e input=35b9defc69d5e5d1]*/ { return PyLong_FromSsize_t(mpd_etop(CTX(self))); } @@ -2997,6 +2997,7 @@ PyDecType_FromSequenceExact(PyTypeObject *type, PyObject *v, PyDecType_FromSequenceExact((st)->PyDec_Type, sequence, context) /*[clinic input] +@permit_long_docstring_body @classmethod _decimal.Decimal.from_float @@ -3022,7 +3023,7 @@ Decimal.from_float(0.1) is not the same as Decimal('0.1'). static PyObject * _decimal_Decimal_from_float_impl(PyTypeObject *type, PyTypeObject *cls, PyObject *pyfloat) -/*[clinic end generated code: output=fcb7d55d2f9dc790 input=03bc8dbe963e52ca]*/ +/*[clinic end generated code: output=fcb7d55d2f9dc790 input=29abf05dd8fe79e4]*/ { PyObject *context; PyObject *result; @@ -3068,6 +3069,8 @@ PyDecType_FromNumberExact(PyTypeObject *type, PyTypeObject *cls, } /*[clinic input] +@permit_long_summary +@permit_long_docstring_body @classmethod _decimal.Decimal.from_number @@ -3088,7 +3091,7 @@ Class method that converts a real number to a decimal number, exactly. static PyObject * _decimal_Decimal_from_number_impl(PyTypeObject *type, PyTypeObject *cls, PyObject *number) -/*[clinic end generated code: output=4d3ec722b7acfd8b input=271cb4feb3148804]*/ +/*[clinic end generated code: output=4d3ec722b7acfd8b input=34ff3696955d3def]*/ { PyObject *context; PyObject *result; @@ -3959,6 +3962,7 @@ dec_as_long(PyObject *dec, PyObject *context, int round) } /*[clinic input] +@permit_long_summary _decimal.Decimal.as_integer_ratio cls: defining_class @@ -3971,7 +3975,7 @@ Raise OverflowError on infinities and a ValueError on NaNs. static PyObject * _decimal_Decimal_as_integer_ratio_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=eb49c512701f844b input=07e33d8852184761]*/ +/*[clinic end generated code: output=eb49c512701f844b input=136f1dc585ca8d80]*/ { PyObject *numerator = NULL; PyObject *denominator = NULL; @@ -4146,17 +4150,17 @@ _decimal.Decimal.to_integral_exact = _decimal.Decimal.to_integral_value Round to the nearest integer. -Decimal.to_integral_exact() signals Inexact or Rounded as appropriate -if rounding occurs. The rounding mode is determined by the rounding -parameter if given, else by the given context. If neither parameter is -given, then the rounding mode of the current default context is used. +This method signals Inexact or Rounded as appropriate if rounding +occurs. The rounding mode is determined by the rounding parameter +if given, else by the given context. If neither parameter is given, +then the rounding mode of the current default context is used. [clinic start generated code]*/ static PyObject * _decimal_Decimal_to_integral_exact_impl(PyObject *self, PyTypeObject *cls, PyObject *rounding, PyObject *context) -/*[clinic end generated code: output=543a39a02eea9917 input=fabce7a744b8087c]*/ +/*[clinic end generated code: output=543a39a02eea9917 input=d4d8abe543393de1]*/ { PyObject *result; uint32_t status = 0; @@ -4791,13 +4795,14 @@ _decimal.Decimal.sqrt = _decimal.Decimal.exp Return the square root of the argument to full precision. -The result is correctly rounded using the ROUND_HALF_EVEN rounding mode. +The result is correctly rounded using the ROUND_HALF_EVEN rounding +mode. [clinic start generated code]*/ static PyObject * _decimal_Decimal_sqrt_impl(PyObject *self, PyTypeObject *cls, PyObject *context) -/*[clinic end generated code: output=deb1280077b5e586 input=3a76afbd39dc20b9]*/ +/*[clinic end generated code: output=deb1280077b5e586 input=c565a7216e9605e7]*/ Dec_UnaryFuncVA(mpd_qsqrt) /* Binary arithmetic functions, optional context arg */ @@ -4853,6 +4858,7 @@ _decimal_Decimal_max_impl(PyObject *self, PyTypeObject *cls, PyObject *other, Dec_BinaryFuncVA(mpd_qmax) /*[clinic input] +@permit_long_summary _decimal.Decimal.max_mag = _decimal.Decimal.compare As the max() method, but compares the absolute values of the operands. @@ -4861,7 +4867,7 @@ As the max() method, but compares the absolute values of the operands. static PyObject * _decimal_Decimal_max_mag_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=f71f2c27d9bc7cac input=88b105e66cf138c5]*/ +/*[clinic end generated code: output=f71f2c27d9bc7cac input=5f81b9da49b45e5d]*/ Dec_BinaryFuncVA(mpd_qmax_mag) /*[clinic input] @@ -4880,6 +4886,7 @@ _decimal_Decimal_min_impl(PyObject *self, PyTypeObject *cls, PyObject *other, Dec_BinaryFuncVA(mpd_qmin) /*[clinic input] +@permit_long_summary _decimal.Decimal.min_mag = _decimal.Decimal.compare As the min() method, but compares the absolute values of the operands. @@ -4888,7 +4895,7 @@ As the min() method, but compares the absolute values of the operands. static PyObject * _decimal_Decimal_min_mag_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=018562ad1c22aae3 input=351fa3c0e592746a]*/ +/*[clinic end generated code: output=018562ad1c22aae3 input=94c29817c7f16db7]*/ Dec_BinaryFuncVA(mpd_qmin_mag) /*[clinic input] @@ -4896,16 +4903,16 @@ _decimal.Decimal.next_toward = _decimal.Decimal.compare Returns the number closest to self, in the direction towards other. -If the two operands are unequal, return the number closest to the first -operand in the direction of the second operand. If both operands are -numerically equal, return a copy of the first operand with the sign set -to be the same as the sign of the second operand. +If the two operands are unequal, return the number closest to the +first operand in the direction of the second operand. If both +operands are numerically equal, return a copy of the first operand +with the sign set to be the same as the sign of the second operand. [clinic start generated code]*/ static PyObject * _decimal_Decimal_next_toward_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=71d879bca8bc1019 input=fdf0091ea6e9e416]*/ +/*[clinic end generated code: output=71d879bca8bc1019 input=adc5d453fc140341]*/ Dec_BinaryFuncVA(mpd_qnext_toward) /*[clinic input] @@ -4914,10 +4921,10 @@ _decimal.Decimal.remainder_near = _decimal.Decimal.compare Return the remainder from dividing self by other. This differs from self % other in that the sign of the remainder is -chosen so as to minimize its absolute value. More precisely, the return -value is self - n * other where n is the integer nearest to the exact -value of self / other, and if two integers are equally near then the -even one is chosen. +chosen so as to minimize its absolute value. More precisely, the +return value is self - n * other where n is the integer nearest to +the exact value of self / other, and if two integers are equally +near then the even one is chosen. If the result is zero then its sign will be the sign of self. [clinic start generated code]*/ @@ -4925,7 +4932,7 @@ If the result is zero then its sign will be the sign of self. static PyObject * _decimal_Decimal_remainder_near_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=d3fbb4985f2077fa input=eb5a8dfe3470b794]*/ +/*[clinic end generated code: output=d3fbb4985f2077fa input=dcb66d4afa0c77c3]*/ Dec_BinaryFuncVA(mpd_qrem_near) /* Ternary arithmetic functions, optional context arg */ @@ -4992,6 +4999,7 @@ _decimal_Decimal_is_infinite_impl(PyObject *self) Dec_BoolFunc(mpd_isinfinite) /*[clinic input] +@permit_long_summary _decimal.Decimal.is_nan Return True if the argument is a (quiet or signaling) NaN, else False. @@ -4999,7 +5007,7 @@ Return True if the argument is a (quiet or signaling) NaN, else False. static PyObject * _decimal_Decimal_is_nan_impl(PyObject *self) -/*[clinic end generated code: output=b704e8b49a164388 input=795e5dac85976994]*/ +/*[clinic end generated code: output=b704e8b49a164388 input=b7d8f0d59fe2332a]*/ Dec_BoolFunc(mpd_isnan) /*[clinic input] @@ -5153,13 +5161,13 @@ _decimal.Decimal.radix Return Decimal(10). -This is the radix (base) in which the Decimal class does -all its arithmetic. Included for compatibility with the specification. +This is the radix (base) in which the Decimal class does all its +arithmetic. Included for compatibility with the specification. [clinic start generated code]*/ static PyObject * _decimal_Decimal_radix_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=40a3bc7ec3d99228 input=b0d4cb9f870bbac1]*/ +/*[clinic end generated code: output=40a3bc7ec3d99228 input=d1cdbdbbbdefdec2]*/ { decimal_state *state = PyType_GetModuleState(cls); return _dec_mpd_radix(state); @@ -5250,15 +5258,15 @@ _decimal.Decimal.logb = _decimal.Decimal.exp Return the adjusted exponent of the operand as a Decimal instance. -If the operand is a zero, then Decimal('-Infinity') is returned and the -DivisionByZero condition is raised. If the operand is an infinity then -Decimal('Infinity') is returned. +If the operand is a zero, then Decimal('-Infinity') is returned and +the DivisionByZero condition is raised. If the operand is an +infinity then Decimal('Infinity') is returned. [clinic start generated code]*/ static PyObject * _decimal_Decimal_logb_impl(PyObject *self, PyTypeObject *cls, PyObject *context) -/*[clinic end generated code: output=36b0bda09e934245 input=a8df027d1b8a2b17]*/ +/*[clinic end generated code: output=36b0bda09e934245 input=eeafa6bbf8d8a013]*/ Dec_UnaryFuncVA(mpd_qlogb) /*[clinic input] @@ -5280,14 +5288,15 @@ The returned value is one of the following ten strings: * '+Normal', indicating that the operand is a positive normal number. * '+Infinity', indicating that the operand is positive infinity. - * 'NaN', indicating that the operand is a quiet NaN (Not a Number). + * 'NaN', indicating that the operand is a quiet NaN (Not a + Number). * 'sNaN', indicating that the operand is a signaling NaN. [clinic start generated code]*/ static PyObject * _decimal_Decimal_number_class_impl(PyObject *self, PyTypeObject *cls, PyObject *context) -/*[clinic end generated code: output=1ac82412e0849c52 input=447095d2677fa0ca]*/ +/*[clinic end generated code: output=1ac82412e0849c52 input=0b59852b43c521aa]*/ { const char *cp; @@ -5303,19 +5312,19 @@ _decimal.Decimal.to_eng_string = _decimal.Decimal.exp Convert to an engineering-type string. -Engineering notation has an exponent which is a multiple of 3, so there -are up to 3 digits left of the decimal place. For example, +Engineering notation has an exponent which is a multiple of 3, so +there are up to 3 digits left of the decimal place. For example, Decimal('123E+1') is converted to Decimal('1.23E+3'). -The value of context.capitals determines whether the exponent sign is -lower or upper case. Otherwise, the context does not affect the +The value of context.capitals determines whether the exponent sign +is lower or upper case. Otherwise, the context does not affect the operation. [clinic start generated code]*/ static PyObject * _decimal_Decimal_to_eng_string_impl(PyObject *self, PyTypeObject *cls, PyObject *context) -/*[clinic end generated code: output=901f128d437ae5c0 input=b2cb7e01e268e45d]*/ +/*[clinic end generated code: output=901f128d437ae5c0 input=111db4de6561f211]*/ { PyObject *result; mpd_ssize_t size; @@ -5343,31 +5352,31 @@ _decimal.Decimal.compare_total = _decimal.Decimal.compare Compare two operands using their abstract representation. -Similar to the compare() method, but the result -gives a total ordering on Decimal instances. Two Decimal instances with -the same numeric value but different representations compare unequal -in this ordering: +Similar to the compare() method, but the result gives a total +ordering on Decimal instances. Two Decimal instances with the same +numeric value but different representations compare unequal in this +ordering: >>> Decimal('12.0').compare_total(Decimal('12')) Decimal('-1') -Quiet and signaling NaNs are also included in the total ordering. The -result of this function is Decimal('0') if both operands have the same -representation, Decimal('-1') if the first operand is lower in the -total order than the second, and Decimal('1') if the first operand is -higher in the total order than the second operand. See the -specification for details of the total order. +Quiet and signaling NaNs are also included in the total ordering. +The result of this function is Decimal('0') if both operands have +the same representation, Decimal('-1') if the first operand is lower +in the total order than the second, and Decimal('1') if the first +operand is higher in the total order than the second operand. See +the specification for details of the total order. This operation is unaffected by context and is quiet: no flags are -changed and no rounding is performed. As an exception, the C version -may raise InvalidOperation if the second operand cannot be converted -exactly. +changed and no rounding is performed. As an exception, the C +version may raise InvalidOperation if the second operand cannot be +converted exactly. [clinic start generated code]*/ static PyObject * _decimal_Decimal_compare_total_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=83649010bad7815f input=6f3111ec5fdbf3c1]*/ +/*[clinic end generated code: output=83649010bad7815f input=d795bf204b9ff2a8]*/ Dec_BinaryFuncVA_NO_CTX(mpd_compare_total) /*[clinic input] @@ -5516,18 +5525,19 @@ _decimal.Decimal.rotate = _decimal.Decimal.compare Returns a rotated copy of self's digits, value-of-other times. -The second operand must be an integer in the range -precision through -precision. The absolute value of the second operand gives the number of -places to rotate. If the second operand is positive then rotation is to -the left; otherwise rotation is to the right. The coefficient of the -first operand is padded on the left with zeros to length precision if -necessary. The sign and exponent of the first operand are unchanged. +The second operand must be an integer in the range -precision +through precision. The absolute value of the second operand gives +the number of places to rotate. If the second operand is positive +then rotation is to the left; otherwise rotation is to the right. +The coefficient of the first operand is padded on the left with +zeros to length precision if necessary. The sign and exponent of +the first operand are unchanged. [clinic start generated code]*/ static PyObject * _decimal_Decimal_rotate_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=09f2737082882b83 input=cde7b032eac43f0b]*/ +/*[clinic end generated code: output=09f2737082882b83 input=4bc840d51842934c]*/ Dec_BinaryFuncVA(mpd_qrotate) /*[clinic input] @@ -5550,18 +5560,18 @@ _decimal.Decimal.shift = _decimal.Decimal.compare Returns a shifted copy of self's digits, value-of-other times. -The second operand must be an integer in the range -precision through -precision. The absolute value of the second operand gives the number -of places to shift. If the second operand is positive, then the shift -is to the left; otherwise the shift is to the right. Digits shifted -into the coefficient are zeros. The sign and exponent of the first -operand are unchanged. +The second operand must be an integer in the range -precision +through precision. The absolute value of the second operand gives +the number of places to shift. If the second operand is positive, +then the shift is to the left; otherwise the shift is to the right. +Digits shifted into the coefficient are zeros. The sign and +exponent of the first operand are unchanged. [clinic start generated code]*/ static PyObject * _decimal_Decimal_shift_impl(PyObject *self, PyTypeObject *cls, PyObject *other, PyObject *context) -/*[clinic end generated code: output=82e061a0d9ecc4f5 input=501759c2522cb78e]*/ +/*[clinic end generated code: output=82e061a0d9ecc4f5 input=c05f3fd69fc1f9f9]*/ Dec_BinaryFuncVA(mpd_qshift) /*[clinic input] @@ -5589,18 +5599,18 @@ that of the right-hand operand. Also unlike other operations, quantize never signals Underflow, even if the result is subnormal and inexact. -If the exponent of the second operand is larger than that of the first, -then rounding may be necessary. In this case, the rounding mode is -determined by the rounding argument if given, else by the given context -argument; if neither argument is given, the rounding mode of the -current thread's context is used. +If the exponent of the second operand is larger than that of the +first, then rounding may be necessary. In this case, the rounding +mode is determined by the rounding argument if given, else by the +given context argument; if neither argument is given, the rounding +mode of the current thread's context is used. [clinic start generated code]*/ static PyObject * _decimal_Decimal_quantize_impl(PyObject *self, PyTypeObject *cls, PyObject *w, PyObject *rounding, PyObject *context) -/*[clinic end generated code: output=fc51edf458559913 input=1166e6311e047b74]*/ +/*[clinic end generated code: output=fc51edf458559913 input=7838b0a5f684adb8]*/ { PyObject *a, *b; PyObject *result; @@ -6629,14 +6639,14 @@ _decimal.Context.remainder_near = _decimal.Context.add Return x - y * n. -Here n is the integer nearest the exact value of x / y (if the result -is 0 then its sign will be the sign of x). +Here n is the integer nearest the exact value of x / y (if the +result is 0 then its sign will be the sign of x). [clinic start generated code]*/ static PyObject * _decimal_Context_remainder_near_impl(PyObject *context, PyTypeObject *cls, PyObject *x, PyObject *y) -/*[clinic end generated code: output=7f18c535a12cf8ac input=bafb6327bb314c5c]*/ +/*[clinic end generated code: output=7f18c535a12cf8ac input=60342558000d4be6]*/ DecCtx_BinaryFunc(mpd_qrem_near) /*[clinic input] @@ -6723,13 +6733,14 @@ restrictions hold: * all three arguments must be integral * 'b' must be nonnegative * at least one of 'a' or 'b' must be nonzero - * modulo must be nonzero and less than 10**prec in absolute value + * modulo must be nonzero and less than 10**prec in absolute + value [clinic start generated code]*/ static PyObject * _decimal_Context_power_impl(PyObject *context, PyTypeObject *cls, PyObject *base, PyObject *exp, PyObject *mod) -/*[clinic end generated code: output=d06d40c37cdd69dc input=2a70edd03317c666]*/ +/*[clinic end generated code: output=d06d40c37cdd69dc input=178a254468ec189b]*/ { PyObject *a, *b, *c = NULL; PyObject *result; @@ -7276,6 +7287,7 @@ _decimal_Context_copy_sign_impl(PyObject *context, PyTypeObject *cls, } /*[clinic input] +@permit_long_docstring_body _decimal.Context.logical_and = _decimal.Context.add Applies the logical operation 'and' between each operand's digits. @@ -7305,7 +7317,7 @@ The operands must be both logical numbers. static PyObject * _decimal_Context_logical_and_impl(PyObject *context, PyTypeObject *cls, PyObject *x, PyObject *y) -/*[clinic end generated code: output=009dfa08ecaa2ac8 input=bcb7d3d6ab7530de]*/ +/*[clinic end generated code: output=009dfa08ecaa2ac8 input=9f8a93a31b9d7088]*/ DecCtx_BinaryFunc(mpd_qand) /*[clinic input] @@ -7342,6 +7354,7 @@ _decimal_Context_logical_or_impl(PyObject *context, PyTypeObject *cls, DecCtx_BinaryFunc(mpd_qor) /*[clinic input] +@permit_long_docstring_body _decimal.Context.logical_xor = _decimal.Context.add Applies the logical operation 'xor' between each operand's digits. @@ -7371,7 +7384,7 @@ The operands must be both logical numbers. static PyObject * _decimal_Context_logical_xor_impl(PyObject *context, PyTypeObject *cls, PyObject *x, PyObject *y) -/*[clinic end generated code: output=23cd81fdcd865d5a input=fcaaf828c1d2d089]*/ +/*[clinic end generated code: output=23cd81fdcd865d5a input=119412854ae58440]*/ DecCtx_BinaryFunc(mpd_qxor) /*[clinic input] diff --git a/Modules/_decimal/clinic/_decimal.c.h b/Modules/_decimal/clinic/_decimal.c.h index b09200845d12e98..c803006ad443825 100644 --- a/Modules/_decimal/clinic/_decimal.c.h +++ b/Modules/_decimal/clinic/_decimal.c.h @@ -36,8 +36,8 @@ PyDoc_STRVAR(_decimal_Context_Etop__doc__, "\n" "Return a value equal to Emax - prec + 1.\n" "\n" -"This is the maximum exponent if the _clamp field of the context is set\n" -"to 1 (IEEE clamp mode). Etop() must not be negative."); +"This is the maximum exponent if the _clamp field of the context is\n" +"set to 1 (IEEE clamp mode). Etop() must not be negative."); #define _DECIMAL_CONTEXT_ETOP_METHODDEF \ {"Etop", (PyCFunction)_decimal_Context_Etop, METH_NOARGS, _decimal_Context_Etop__doc__}, @@ -1092,10 +1092,10 @@ PyDoc_STRVAR(_decimal_Decimal_to_integral_exact__doc__, "\n" "Round to the nearest integer.\n" "\n" -"Decimal.to_integral_exact() signals Inexact or Rounded as appropriate\n" -"if rounding occurs. The rounding mode is determined by the rounding\n" -"parameter if given, else by the given context. If neither parameter is\n" -"given, then the rounding mode of the current default context is used."); +"This method signals Inexact or Rounded as appropriate if rounding\n" +"occurs. The rounding mode is determined by the rounding parameter\n" +"if given, else by the given context. If neither parameter is given,\n" +"then the rounding mode of the current default context is used."); #define _DECIMAL_DECIMAL_TO_INTEGRAL_EXACT_METHODDEF \ {"to_integral_exact", _PyCFunction_CAST(_decimal_Decimal_to_integral_exact), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_to_integral_exact__doc__}, @@ -1638,7 +1638,8 @@ PyDoc_STRVAR(_decimal_Decimal_sqrt__doc__, "\n" "Return the square root of the argument to full precision.\n" "\n" -"The result is correctly rounded using the ROUND_HALF_EVEN rounding mode."); +"The result is correctly rounded using the ROUND_HALF_EVEN rounding\n" +"mode."); #define _DECIMAL_DECIMAL_SQRT_METHODDEF \ {"sqrt", _PyCFunction_CAST(_decimal_Decimal_sqrt), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_sqrt__doc__}, @@ -2113,10 +2114,10 @@ PyDoc_STRVAR(_decimal_Decimal_next_toward__doc__, "\n" "Returns the number closest to self, in the direction towards other.\n" "\n" -"If the two operands are unequal, return the number closest to the first\n" -"operand in the direction of the second operand. If both operands are\n" -"numerically equal, return a copy of the first operand with the sign set\n" -"to be the same as the sign of the second operand."); +"If the two operands are unequal, return the number closest to the\n" +"first operand in the direction of the second operand. If both\n" +"operands are numerically equal, return a copy of the first operand\n" +"with the sign set to be the same as the sign of the second operand."); #define _DECIMAL_DECIMAL_NEXT_TOWARD_METHODDEF \ {"next_toward", _PyCFunction_CAST(_decimal_Decimal_next_toward), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_next_toward__doc__}, @@ -2185,10 +2186,10 @@ PyDoc_STRVAR(_decimal_Decimal_remainder_near__doc__, "Return the remainder from dividing self by other.\n" "\n" "This differs from self % other in that the sign of the remainder is\n" -"chosen so as to minimize its absolute value. More precisely, the return\n" -"value is self - n * other where n is the integer nearest to the exact\n" -"value of self / other, and if two integers are equally near then the\n" -"even one is chosen.\n" +"chosen so as to minimize its absolute value. More precisely, the\n" +"return value is self - n * other where n is the integer nearest to\n" +"the exact value of self / other, and if two integers are equally\n" +"near then the even one is chosen.\n" "\n" "If the result is zero then its sign will be the sign of self."); @@ -2671,8 +2672,8 @@ PyDoc_STRVAR(_decimal_Decimal_radix__doc__, "\n" "Return Decimal(10).\n" "\n" -"This is the radix (base) in which the Decimal class does\n" -"all its arithmetic. Included for compatibility with the specification."); +"This is the radix (base) in which the Decimal class does all its\n" +"arithmetic. Included for compatibility with the specification."); #define _DECIMAL_DECIMAL_RADIX_METHODDEF \ {"radix", _PyCFunction_CAST(_decimal_Decimal_radix), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_radix__doc__}, @@ -2812,9 +2813,9 @@ PyDoc_STRVAR(_decimal_Decimal_logb__doc__, "\n" "Return the adjusted exponent of the operand as a Decimal instance.\n" "\n" -"If the operand is a zero, then Decimal(\'-Infinity\') is returned and the\n" -"DivisionByZero condition is raised. If the operand is an infinity then\n" -"Decimal(\'Infinity\') is returned."); +"If the operand is a zero, then Decimal(\'-Infinity\') is returned and\n" +"the DivisionByZero condition is raised. If the operand is an\n" +"infinity then Decimal(\'Infinity\') is returned."); #define _DECIMAL_DECIMAL_LOGB_METHODDEF \ {"logb", _PyCFunction_CAST(_decimal_Decimal_logb), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_logb__doc__}, @@ -2894,7 +2895,8 @@ PyDoc_STRVAR(_decimal_Decimal_number_class__doc__, " * \'+Normal\', indicating that the operand is a positive normal\n" " number.\n" " * \'+Infinity\', indicating that the operand is positive infinity.\n" -" * \'NaN\', indicating that the operand is a quiet NaN (Not a Number).\n" +" * \'NaN\', indicating that the operand is a quiet NaN (Not a\n" +" Number).\n" " * \'sNaN\', indicating that the operand is a signaling NaN."); #define _DECIMAL_DECIMAL_NUMBER_CLASS_METHODDEF \ @@ -2961,12 +2963,12 @@ PyDoc_STRVAR(_decimal_Decimal_to_eng_string__doc__, "\n" "Convert to an engineering-type string.\n" "\n" -"Engineering notation has an exponent which is a multiple of 3, so there\n" -"are up to 3 digits left of the decimal place. For example,\n" +"Engineering notation has an exponent which is a multiple of 3, so\n" +"there are up to 3 digits left of the decimal place. For example,\n" "Decimal(\'123E+1\') is converted to Decimal(\'1.23E+3\').\n" "\n" -"The value of context.capitals determines whether the exponent sign is\n" -"lower or upper case. Otherwise, the context does not affect the\n" +"The value of context.capitals determines whether the exponent sign\n" +"is lower or upper case. Otherwise, the context does not affect the\n" "operation."); #define _DECIMAL_DECIMAL_TO_ENG_STRING_METHODDEF \ @@ -3033,25 +3035,25 @@ PyDoc_STRVAR(_decimal_Decimal_compare_total__doc__, "\n" "Compare two operands using their abstract representation.\n" "\n" -"Similar to the compare() method, but the result\n" -"gives a total ordering on Decimal instances. Two Decimal instances with\n" -"the same numeric value but different representations compare unequal\n" -"in this ordering:\n" +"Similar to the compare() method, but the result gives a total\n" +"ordering on Decimal instances. Two Decimal instances with the same\n" +"numeric value but different representations compare unequal in this\n" +"ordering:\n" "\n" " >>> Decimal(\'12.0\').compare_total(Decimal(\'12\'))\n" " Decimal(\'-1\')\n" "\n" -"Quiet and signaling NaNs are also included in the total ordering. The\n" -"result of this function is Decimal(\'0\') if both operands have the same\n" -"representation, Decimal(\'-1\') if the first operand is lower in the\n" -"total order than the second, and Decimal(\'1\') if the first operand is\n" -"higher in the total order than the second operand. See the\n" -"specification for details of the total order.\n" +"Quiet and signaling NaNs are also included in the total ordering.\n" +"The result of this function is Decimal(\'0\') if both operands have\n" +"the same representation, Decimal(\'-1\') if the first operand is lower\n" +"in the total order than the second, and Decimal(\'1\') if the first\n" +"operand is higher in the total order than the second operand. See\n" +"the specification for details of the total order.\n" "\n" "This operation is unaffected by context and is quiet: no flags are\n" -"changed and no rounding is performed. As an exception, the C version\n" -"may raise InvalidOperation if the second operand cannot be converted\n" -"exactly."); +"changed and no rounding is performed. As an exception, the C\n" +"version may raise InvalidOperation if the second operand cannot be\n" +"converted exactly."); #define _DECIMAL_DECIMAL_COMPARE_TOTAL_METHODDEF \ {"compare_total", _PyCFunction_CAST(_decimal_Decimal_compare_total), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_compare_total__doc__}, @@ -3544,12 +3546,13 @@ PyDoc_STRVAR(_decimal_Decimal_rotate__doc__, "\n" "Returns a rotated copy of self\'s digits, value-of-other times.\n" "\n" -"The second operand must be an integer in the range -precision through\n" -"precision. The absolute value of the second operand gives the number of\n" -"places to rotate. If the second operand is positive then rotation is to\n" -"the left; otherwise rotation is to the right. The coefficient of the\n" -"first operand is padded on the left with zeros to length precision if\n" -"necessary. The sign and exponent of the first operand are unchanged."); +"The second operand must be an integer in the range -precision\n" +"through precision. The absolute value of the second operand gives\n" +"the number of places to rotate. If the second operand is positive\n" +"then rotation is to the left; otherwise rotation is to the right.\n" +"The coefficient of the first operand is padded on the left with\n" +"zeros to length precision if necessary. The sign and exponent of\n" +"the first operand are unchanged."); #define _DECIMAL_DECIMAL_ROTATE_METHODDEF \ {"rotate", _PyCFunction_CAST(_decimal_Decimal_rotate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_rotate__doc__}, @@ -3686,12 +3689,12 @@ PyDoc_STRVAR(_decimal_Decimal_shift__doc__, "\n" "Returns a shifted copy of self\'s digits, value-of-other times.\n" "\n" -"The second operand must be an integer in the range -precision through\n" -"precision. The absolute value of the second operand gives the number\n" -"of places to shift. If the second operand is positive, then the shift\n" -"is to the left; otherwise the shift is to the right. Digits shifted\n" -"into the coefficient are zeros. The sign and exponent of the first\n" -"operand are unchanged."); +"The second operand must be an integer in the range -precision\n" +"through precision. The absolute value of the second operand gives\n" +"the number of places to shift. If the second operand is positive,\n" +"then the shift is to the left; otherwise the shift is to the right.\n" +"Digits shifted into the coefficient are zeros. The sign and\n" +"exponent of the first operand are unchanged."); #define _DECIMAL_DECIMAL_SHIFT_METHODDEF \ {"shift", _PyCFunction_CAST(_decimal_Decimal_shift), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_shift__doc__}, @@ -3774,11 +3777,11 @@ PyDoc_STRVAR(_decimal_Decimal_quantize__doc__, "Also unlike other operations, quantize never signals Underflow, even\n" "if the result is subnormal and inexact.\n" "\n" -"If the exponent of the second operand is larger than that of the first,\n" -"then rounding may be necessary. In this case, the rounding mode is\n" -"determined by the rounding argument if given, else by the given context\n" -"argument; if neither argument is given, the rounding mode of the\n" -"current thread\'s context is used."); +"If the exponent of the second operand is larger than that of the\n" +"first, then rounding may be necessary. In this case, the rounding\n" +"mode is determined by the rounding argument if given, else by the\n" +"given context argument; if neither argument is given, the rounding\n" +"mode of the current thread\'s context is used."); #define _DECIMAL_DECIMAL_QUANTIZE_METHODDEF \ {"quantize", _PyCFunction_CAST(_decimal_Decimal_quantize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Decimal_quantize__doc__}, @@ -5197,8 +5200,8 @@ PyDoc_STRVAR(_decimal_Context_remainder_near__doc__, "\n" "Return x - y * n.\n" "\n" -"Here n is the integer nearest the exact value of x / y (if the result\n" -"is 0 then its sign will be the sign of x)."); +"Here n is the integer nearest the exact value of x / y (if the\n" +"result is 0 then its sign will be the sign of x)."); #define _DECIMAL_CONTEXT_REMAINDER_NEAR_METHODDEF \ {"remainder_near", _PyCFunction_CAST(_decimal_Context_remainder_near), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_remainder_near__doc__}, @@ -5336,7 +5339,8 @@ PyDoc_STRVAR(_decimal_Context_power__doc__, " * all three arguments must be integral\n" " * \'b\' must be nonnegative\n" " * at least one of \'a\' or \'b\' must be nonzero\n" -" * modulo must be nonzero and less than 10**prec in absolute value"); +" * modulo must be nonzero and less than 10**prec in absolute\n" +" value"); #define _DECIMAL_CONTEXT_POWER_METHODDEF \ {"power", _PyCFunction_CAST(_decimal_Context_power), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _decimal_Context_power__doc__}, @@ -6980,4 +6984,4 @@ _decimal_Context_same_quantum(PyObject *context, PyTypeObject *cls, PyObject *co #ifndef _DECIMAL_CONTEXT_APPLY_METHODDEF #define _DECIMAL_CONTEXT_APPLY_METHODDEF #endif /* !defined(_DECIMAL_CONTEXT_APPLY_METHODDEF) */ -/*[clinic end generated code: output=b288181c82fdc9f1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0eb835634388294e input=a9049054013a1b77]*/ diff --git a/Modules/_decimal/libmpdec/README.txt b/Modules/_decimal/libmpdec/README.txt deleted file mode 100644 index c1d481dee764523..000000000000000 --- a/Modules/_decimal/libmpdec/README.txt +++ /dev/null @@ -1,88 +0,0 @@ - - -libmpdec -======== - -libmpdec is a fast C/C++ library for correctly-rounded arbitrary precision -decimal floating point arithmetic. It is a complete implementation of -Mike Cowlishaw/IBM's General Decimal Arithmetic Specification. - - -Files required for the Python _decimal module -============================================= - - Core files for small and medium precision arithmetic - ---------------------------------------------------- - - basearith.{c,h} -> Core arithmetic in base 10**9 or 10**19. - bits.h -> Portable detection of least/most significant one-bit. - constants.{c,h} -> Constants that are used in multiple files. - context.c -> Context functions. - io.{c,h} -> Conversions between mpd_t and ASCII strings, - mpd_t formatting (allows UTF-8 fill character). - mpalloc.{c,h} -> Allocation handlers with overflow detection - and functions for switching between static - and dynamic mpd_t. - mpdecimal.{c,h} -> All (quiet) functions of the specification. - typearith.h -> Fast primitives for double word multiplication, - division etc. - - Visual Studio only: - ~~~~~~~~~~~~~~~~~~~ - vcdiv64.asm -> Double word division used in typearith.h. VS 2008 does - not allow inline asm for x64. Also, it does not provide - an intrinsic for double word division. - - Files for bignum arithmetic: - ---------------------------- - - The following files implement the Fast Number Theoretic Transform - used for multiplying coefficients with more than 1024 words (see - mpdecimal.c: _mpd_fntmul()). - - umodarith.h -> Fast low level routines for unsigned modular arithmetic. - numbertheory.{c,h} -> Routines for setting up the Number Theoretic Transform. - difradix2.{c,h} -> Decimation in frequency transform, used as the - "base case" by the following three files: - - fnt.{c,h} -> Transform arrays up to 4096 words. - sixstep.{c,h} -> Transform larger arrays of length 2**n. - fourstep.{c,h} -> Transform larger arrays of length 3 * 2**n. - - convolute.{c,h} -> Fast convolution using one of the three transform - functions. - transpose.{c,h} -> Transpositions needed for the sixstep algorithm. - crt.{c,h} -> Chinese Remainder Theorem: use information from three - transforms modulo three different primes to get the - final result. - - -Pointers to literature, proofs and more -======================================= - - literature/ - ----------- - - REFERENCES.txt -> List of relevant papers. - bignum.txt -> Explanation of the Fast Number Theoretic Transform (FNT). - fnt.py -> Verify constants used in the FNT; Python demo for the - O(N**2) discrete transform. - - matrix-transform.txt -> Proof for the Matrix Fourier Transform used in - fourstep.c. - six-step.txt -> Show that the algorithm used in sixstep.c is - a variant of the Matrix Fourier Transform. - mulmod-64.txt -> Proof for the mulmod64 algorithm from - umodarith.h. - mulmod-ppro.txt -> Proof for the x87 FPU modular multiplication - from umodarith.h. - umodarith.lisp -> ACL2 proofs for many functions from umodarith.h. - - -Library Author -============== - - Stefan Krah <skrah@bytereef.org> - - - diff --git a/Modules/_decimal/libmpdec/basearith.c b/Modules/_decimal/libmpdec/basearith.c deleted file mode 100644 index 85c608fadf51567..000000000000000 --- a/Modules/_decimal/libmpdec/basearith.c +++ /dev/null @@ -1,655 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> -#include <stdio.h> - -#include "basearith.h" -#include "constants.h" -#include "typearith.h" - - -/*********************************************************************/ -/* Calculations in base MPD_RADIX */ -/*********************************************************************/ - - -/* - * Knuth, TAOCP, Volume 2, 4.3.1: - * w := sum of u (len m) and v (len n) - * n > 0 and m >= n - * The calling function has to handle a possible final carry. - */ -mpd_uint_t -_mpd_baseadd(mpd_uint_t *w, const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t m, mpd_size_t n) -{ - mpd_uint_t s; - mpd_uint_t carry = 0; - mpd_size_t i; - - assert(n > 0 && m >= n); - - /* add n members of u and v */ - for (i = 0; i < n; i++) { - s = u[i] + (v[i] + carry); - carry = (s < u[i]) | (s >= MPD_RADIX); - w[i] = carry ? s-MPD_RADIX : s; - } - /* if there is a carry, propagate it */ - for (; carry && i < m; i++) { - s = u[i] + carry; - carry = (s == MPD_RADIX); - w[i] = carry ? 0 : s; - } - /* copy the rest of u */ - for (; i < m; i++) { - w[i] = u[i]; - } - - return carry; -} - -/* - * Add the contents of u to w. Carries are propagated further. The caller - * has to make sure that w is big enough. - */ -void -_mpd_baseaddto(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n) -{ - mpd_uint_t s; - mpd_uint_t carry = 0; - mpd_size_t i; - - if (n == 0) return; - - /* add n members of u to w */ - for (i = 0; i < n; i++) { - s = w[i] + (u[i] + carry); - carry = (s < w[i]) | (s >= MPD_RADIX); - w[i] = carry ? s-MPD_RADIX : s; - } - /* if there is a carry, propagate it */ - for (; carry; i++) { - s = w[i] + carry; - carry = (s == MPD_RADIX); - w[i] = carry ? 0 : s; - } -} - -/* - * Add v to w (len m). The calling function has to handle a possible - * final carry. Assumption: m > 0. - */ -mpd_uint_t -_mpd_shortadd(mpd_uint_t *w, mpd_size_t m, mpd_uint_t v) -{ - mpd_uint_t s; - mpd_uint_t carry; - mpd_size_t i; - - assert(m > 0); - - /* add v to w */ - s = w[0] + v; - carry = (s < v) | (s >= MPD_RADIX); - w[0] = carry ? s-MPD_RADIX : s; - - /* if there is a carry, propagate it */ - for (i = 1; carry && i < m; i++) { - s = w[i] + carry; - carry = (s == MPD_RADIX); - w[i] = carry ? 0 : s; - } - - return carry; -} - -/* Increment u. The calling function has to handle a possible carry. */ -mpd_uint_t -_mpd_baseincr(mpd_uint_t *u, mpd_size_t n) -{ - mpd_uint_t s; - mpd_uint_t carry = 1; - mpd_size_t i; - - assert(n > 0); - - /* if there is a carry, propagate it */ - for (i = 0; carry && i < n; i++) { - s = u[i] + carry; - carry = (s == MPD_RADIX); - u[i] = carry ? 0 : s; - } - - return carry; -} - -/* - * Knuth, TAOCP, Volume 2, 4.3.1: - * w := difference of u (len m) and v (len n). - * number in u >= number in v; - */ -void -_mpd_basesub(mpd_uint_t *w, const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t m, mpd_size_t n) -{ - mpd_uint_t d; - mpd_uint_t borrow = 0; - mpd_size_t i; - - assert(m > 0 && n > 0); - - /* subtract n members of v from u */ - for (i = 0; i < n; i++) { - d = u[i] - (v[i] + borrow); - borrow = (u[i] < d); - w[i] = borrow ? d + MPD_RADIX : d; - } - /* if there is a borrow, propagate it */ - for (; borrow && i < m; i++) { - d = u[i] - borrow; - borrow = (u[i] == 0); - w[i] = borrow ? MPD_RADIX-1 : d; - } - /* copy the rest of u */ - for (; i < m; i++) { - w[i] = u[i]; - } -} - -/* - * Subtract the contents of u from w. w is larger than u. Borrows are - * propagated further, but eventually w can absorb the final borrow. - */ -void -_mpd_basesubfrom(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n) -{ - mpd_uint_t d; - mpd_uint_t borrow = 0; - mpd_size_t i; - - if (n == 0) return; - - /* subtract n members of u from w */ - for (i = 0; i < n; i++) { - d = w[i] - (u[i] + borrow); - borrow = (w[i] < d); - w[i] = borrow ? d + MPD_RADIX : d; - } - /* if there is a borrow, propagate it */ - for (; borrow; i++) { - d = w[i] - borrow; - borrow = (w[i] == 0); - w[i] = borrow ? MPD_RADIX-1 : d; - } -} - -/* w := product of u (len n) and v (single word) */ -void -_mpd_shortmul(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, mpd_uint_t v) -{ - mpd_uint_t hi, lo; - mpd_uint_t carry = 0; - mpd_size_t i; - - assert(n > 0); - - for (i=0; i < n; i++) { - - _mpd_mul_words(&hi, &lo, u[i], v); - lo = carry + lo; - if (lo < carry) hi++; - - _mpd_div_words_r(&carry, &w[i], hi, lo); - } - w[i] = carry; -} - -/* - * Knuth, TAOCP, Volume 2, 4.3.1: - * w := product of u (len m) and v (len n) - * w must be initialized to zero - */ -void -_mpd_basemul(mpd_uint_t *w, const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t m, mpd_size_t n) -{ - mpd_uint_t hi, lo; - mpd_uint_t carry; - mpd_size_t i, j; - - assert(m > 0 && n > 0); - - for (j=0; j < n; j++) { - carry = 0; - for (i=0; i < m; i++) { - - _mpd_mul_words(&hi, &lo, u[i], v[j]); - lo = w[i+j] + lo; - if (lo < w[i+j]) hi++; - lo = carry + lo; - if (lo < carry) hi++; - - _mpd_div_words_r(&carry, &w[i+j], hi, lo); - } - w[j+m] = carry; - } -} - -/* - * Knuth, TAOCP Volume 2, 4.3.1, exercise 16: - * w := quotient of u (len n) divided by a single word v - */ -mpd_uint_t -_mpd_shortdiv(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, mpd_uint_t v) -{ - mpd_uint_t hi, lo; - mpd_uint_t rem = 0; - mpd_size_t i; - - assert(n > 0); - - for (i=n-1; i != MPD_SIZE_MAX; i--) { - - _mpd_mul_words(&hi, &lo, rem, MPD_RADIX); - lo = u[i] + lo; - if (lo < u[i]) hi++; - - _mpd_div_words(&w[i], &rem, hi, lo, v); - } - - return rem; -} - -/* - * Knuth, TAOCP Volume 2, 4.3.1: - * q, r := quotient and remainder of uconst (len nplusm) - * divided by vconst (len n) - * nplusm >= n - * - * If r is not NULL, r will contain the remainder. If r is NULL, the - * return value indicates if there is a remainder: 1 for true, 0 for - * false. A return value of -1 indicates an error. - */ -int -_mpd_basedivmod(mpd_uint_t *q, mpd_uint_t *r, - const mpd_uint_t *uconst, const mpd_uint_t *vconst, - mpd_size_t nplusm, mpd_size_t n) -{ - mpd_uint_t ustatic[MPD_MINALLOC_MAX]; - mpd_uint_t vstatic[MPD_MINALLOC_MAX]; - mpd_uint_t *u = ustatic; - mpd_uint_t *v = vstatic; - mpd_uint_t d, qhat, rhat, w2[2]; - mpd_uint_t hi, lo, x; - mpd_uint_t carry; - mpd_size_t i, j, m; - int retval = 0; - - assert(n > 1 && nplusm >= n); - m = sub_size_t(nplusm, n); - - /* D1: normalize */ - d = MPD_RADIX / (vconst[n-1] + 1); - - if (nplusm >= MPD_MINALLOC_MAX) { - if ((u = mpd_alloc(nplusm+1, sizeof *u)) == NULL) { - return -1; - } - } - if (n >= MPD_MINALLOC_MAX) { - if ((v = mpd_alloc(n+1, sizeof *v)) == NULL) { - mpd_free(u); - return -1; - } - } - - _mpd_shortmul(u, uconst, nplusm, d); - _mpd_shortmul(v, vconst, n, d); - - /* D2: loop */ - for (j=m; j != MPD_SIZE_MAX; j--) { - assert(2 <= j+n && j+n <= nplusm); /* annotation for scan-build */ - - /* D3: calculate qhat and rhat */ - rhat = _mpd_shortdiv(w2, u+j+n-1, 2, v[n-1]); - qhat = w2[1] * MPD_RADIX + w2[0]; - - while (1) { - if (qhat < MPD_RADIX) { - _mpd_singlemul(w2, qhat, v[n-2]); - if (w2[1] <= rhat) { - if (w2[1] != rhat || w2[0] <= u[j+n-2]) { - break; - } - } - } - qhat -= 1; - rhat += v[n-1]; - if (rhat < v[n-1] || rhat >= MPD_RADIX) { - break; - } - } - /* D4: multiply and subtract */ - carry = 0; - for (i=0; i <= n; i++) { - - _mpd_mul_words(&hi, &lo, qhat, v[i]); - - lo = carry + lo; - if (lo < carry) hi++; - - _mpd_div_words_r(&hi, &lo, hi, lo); - - x = u[i+j] - lo; - carry = (u[i+j] < x); - u[i+j] = carry ? x+MPD_RADIX : x; - carry += hi; - } - q[j] = qhat; - /* D5: test remainder */ - if (carry) { - q[j] -= 1; - /* D6: add back */ - (void)_mpd_baseadd(u+j, u+j, v, n+1, n); - } - } - - /* D8: unnormalize */ - if (r != NULL) { - _mpd_shortdiv(r, u, n, d); - /* we are not interested in the return value here */ - retval = 0; - } - else { - retval = !_mpd_isallzero(u, n); - } - - -if (u != ustatic) mpd_free(u); -if (v != vstatic) mpd_free(v); -return retval; -} - -/* - * Left shift of src by 'shift' digits; src may equal dest. - * - * dest := area of n mpd_uint_t with space for srcdigits+shift digits. - * src := coefficient with length m. - * - * The case splits in the function are non-obvious. The following - * equations might help: - * - * Let msdigits denote the number of digits in the most significant - * word of src. Then 1 <= msdigits <= rdigits. - * - * 1) shift = q * rdigits + r - * 2) srcdigits = qsrc * rdigits + msdigits - * 3) destdigits = shift + srcdigits - * = q * rdigits + r + qsrc * rdigits + msdigits - * = q * rdigits + (qsrc * rdigits + (r + msdigits)) - * - * The result has q zero words, followed by the coefficient that - * is left-shifted by r. The case r == 0 is trivial. For r > 0, it - * is important to keep in mind that we always read m source words, - * but write m+1 destination words if r + msdigits > rdigits, m words - * otherwise. - */ -void -_mpd_baseshiftl(mpd_uint_t *dest, mpd_uint_t *src, mpd_size_t n, mpd_size_t m, - mpd_size_t shift) -{ -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && !defined(__clang__) - /* spurious uninitialized warnings */ - mpd_uint_t l=l, lprev=lprev, h=h; -#else - mpd_uint_t l, lprev, h; -#endif - mpd_uint_t q, r; - mpd_uint_t ph; - - assert(m > 0 && n >= m); - - _mpd_div_word(&q, &r, (mpd_uint_t)shift, MPD_RDIGITS); - - if (r != 0) { - - ph = mpd_pow10[r]; - - --m; --n; - _mpd_divmod_pow10(&h, &lprev, src[m--], MPD_RDIGITS-r); - if (h != 0) { /* r + msdigits > rdigits <==> h != 0 */ - dest[n--] = h; - } - /* write m-1 shifted words */ - for (; m != MPD_SIZE_MAX; m--,n--) { - _mpd_divmod_pow10(&h, &l, src[m], MPD_RDIGITS-r); - dest[n] = ph * lprev + h; - lprev = l; - } - /* write least significant word */ - dest[q] = ph * lprev; - } - else { - while (--m != MPD_SIZE_MAX) { - dest[m+q] = src[m]; - } - } - - mpd_uint_zero(dest, q); -} - -/* - * Right shift of src by 'shift' digits; src may equal dest. - * Assumption: srcdigits-shift > 0. - * - * dest := area with space for srcdigits-shift digits. - * src := coefficient with length 'slen'. - * - * The case splits in the function rely on the following equations: - * - * Let msdigits denote the number of digits in the most significant - * word of src. Then 1 <= msdigits <= rdigits. - * - * 1) shift = q * rdigits + r - * 2) srcdigits = qsrc * rdigits + msdigits - * 3) destdigits = srcdigits - shift - * = qsrc * rdigits + msdigits - (q * rdigits + r) - * = (qsrc - q) * rdigits + msdigits - r - * - * Since destdigits > 0 and 1 <= msdigits <= rdigits: - * - * 4) qsrc >= q - * 5) qsrc == q ==> msdigits > r - * - * The result has slen-q words if msdigits > r, slen-q-1 words otherwise. - */ -mpd_uint_t -_mpd_baseshiftr(mpd_uint_t *dest, mpd_uint_t *src, mpd_size_t slen, - mpd_size_t shift) -{ -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && !defined(__clang__) - /* spurious uninitialized warnings */ - mpd_uint_t l=l, h=h, hprev=hprev; /* low, high, previous high */ -#else - mpd_uint_t l, h, hprev; /* low, high, previous high */ -#endif - mpd_uint_t rnd, rest; /* rounding digit, rest */ - mpd_uint_t q, r; - mpd_size_t i, j; - mpd_uint_t ph; - - assert(slen > 0); - - _mpd_div_word(&q, &r, (mpd_uint_t)shift, MPD_RDIGITS); - - rnd = rest = 0; - if (r != 0) { - - ph = mpd_pow10[MPD_RDIGITS-r]; - - _mpd_divmod_pow10(&hprev, &rest, src[q], r); - _mpd_divmod_pow10(&rnd, &rest, rest, r-1); - - if (rest == 0 && q > 0) { - rest = !_mpd_isallzero(src, q); - } - /* write slen-q-1 words */ - for (j=0,i=q+1; i<slen; i++,j++) { - _mpd_divmod_pow10(&h, &l, src[i], r); - dest[j] = ph * l + hprev; - hprev = h; - } - /* write most significant word */ - if (hprev != 0) { /* always the case if slen==q-1 */ - dest[j] = hprev; - } - } - else { - if (q > 0) { - _mpd_divmod_pow10(&rnd, &rest, src[q-1], MPD_RDIGITS-1); - /* is there any non-zero digit below rnd? */ - if (rest == 0) rest = !_mpd_isallzero(src, q-1); - } - for (j = 0; j < slen-q; j++) { - dest[j] = src[q+j]; - } - } - - /* 0-4 ==> rnd+rest < 0.5 */ - /* 5 ==> rnd+rest == 0.5 */ - /* 6-9 ==> rnd+rest > 0.5 */ - return (rnd == 0 || rnd == 5) ? rnd + !!rest : rnd; -} - - -/*********************************************************************/ -/* Calculations in base b */ -/*********************************************************************/ - -/* - * Add v to w (len m). The calling function has to handle a possible - * final carry. Assumption: m > 0. - */ -mpd_uint_t -_mpd_shortadd_b(mpd_uint_t *w, mpd_size_t m, mpd_uint_t v, mpd_uint_t b) -{ - mpd_uint_t s; - mpd_uint_t carry; - mpd_size_t i; - - assert(m > 0); - - /* add v to w */ - s = w[0] + v; - carry = (s < v) | (s >= b); - w[0] = carry ? s-b : s; - - /* if there is a carry, propagate it */ - for (i = 1; carry && i < m; i++) { - s = w[i] + carry; - carry = (s == b); - w[i] = carry ? 0 : s; - } - - return carry; -} - -/* w := product of u (len n) and v (single word). Return carry. */ -mpd_uint_t -_mpd_shortmul_c(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, mpd_uint_t v) -{ - mpd_uint_t hi, lo; - mpd_uint_t carry = 0; - mpd_size_t i; - - assert(n > 0); - - for (i=0; i < n; i++) { - - _mpd_mul_words(&hi, &lo, u[i], v); - lo = carry + lo; - if (lo < carry) hi++; - - _mpd_div_words_r(&carry, &w[i], hi, lo); - } - - return carry; -} - -/* w := product of u (len n) and v (single word) */ -mpd_uint_t -_mpd_shortmul_b(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, - mpd_uint_t v, mpd_uint_t b) -{ - mpd_uint_t hi, lo; - mpd_uint_t carry = 0; - mpd_size_t i; - - assert(n > 0); - - for (i=0; i < n; i++) { - - _mpd_mul_words(&hi, &lo, u[i], v); - lo = carry + lo; - if (lo < carry) hi++; - - _mpd_div_words(&carry, &w[i], hi, lo, b); - } - - return carry; -} - -/* - * Knuth, TAOCP Volume 2, 4.3.1, exercise 16: - * w := quotient of u (len n) divided by a single word v - */ -mpd_uint_t -_mpd_shortdiv_b(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, - mpd_uint_t v, mpd_uint_t b) -{ - mpd_uint_t hi, lo; - mpd_uint_t rem = 0; - mpd_size_t i; - - assert(n > 0); - - for (i=n-1; i != MPD_SIZE_MAX; i--) { - - _mpd_mul_words(&hi, &lo, rem, b); - lo = u[i] + lo; - if (lo < u[i]) hi++; - - _mpd_div_words(&w[i], &rem, hi, lo, v); - } - - return rem; -} diff --git a/Modules/_decimal/libmpdec/basearith.h b/Modules/_decimal/libmpdec/basearith.h deleted file mode 100644 index d35925aaddb48e0..000000000000000 --- a/Modules/_decimal/libmpdec/basearith.h +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_BASEARITH_H_ -#define LIBMPDEC_BASEARITH_H_ - - -#include "mpdecimal.h" -#include "typearith.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -mpd_uint_t _mpd_baseadd(mpd_uint_t *w, const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t m, mpd_size_t n); -void _mpd_baseaddto(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n); -mpd_uint_t _mpd_shortadd(mpd_uint_t *w, mpd_size_t m, mpd_uint_t v); -mpd_uint_t _mpd_shortadd_b(mpd_uint_t *w, mpd_size_t m, mpd_uint_t v, - mpd_uint_t b); -mpd_uint_t _mpd_baseincr(mpd_uint_t *u, mpd_size_t n); -void _mpd_basesub(mpd_uint_t *w, const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t m, mpd_size_t n); -void _mpd_basesubfrom(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n); -void _mpd_basemul(mpd_uint_t *w, const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t m, mpd_size_t n); -void _mpd_shortmul(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, - mpd_uint_t v); -mpd_uint_t _mpd_shortmul_c(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, - mpd_uint_t v); -mpd_uint_t _mpd_shortmul_b(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, - mpd_uint_t v, mpd_uint_t b); -mpd_uint_t _mpd_shortdiv(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, - mpd_uint_t v); -mpd_uint_t _mpd_shortdiv_b(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, - mpd_uint_t v, mpd_uint_t b); -int _mpd_basedivmod(mpd_uint_t *q, mpd_uint_t *r, const mpd_uint_t *uconst, - const mpd_uint_t *vconst, mpd_size_t nplusm, mpd_size_t n); -void _mpd_baseshiftl(mpd_uint_t *dest, mpd_uint_t *src, mpd_size_t n, - mpd_size_t m, mpd_size_t shift); -mpd_uint_t _mpd_baseshiftr(mpd_uint_t *dest, mpd_uint_t *src, mpd_size_t slen, - mpd_size_t shift); - - - -#ifdef CONFIG_64 -extern const mpd_uint_t mprime_rdx; - -/* - * Algorithm from: Division by Invariant Integers using Multiplication, - * T. Granlund and P. L. Montgomery, Proceedings of the SIGPLAN '94 - * Conference on Programming Language Design and Implementation. - * - * http://gmplib.org/~tege/divcnst-pldi94.pdf - * - * Variables from the paper and their translations (See section 8): - * - * N := 64 - * d := MPD_RADIX - * l := 64 - * m' := floor((2**(64+64) - 1)/MPD_RADIX) - 2**64 - * - * Since N-l == 0: - * - * dnorm := d - * n2 := hi - * n10 := lo - * - * ACL2 proof: mpd-div-words-r-correct - */ -static inline void -_mpd_div_words_r(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t hi, mpd_uint_t lo) -{ - mpd_uint_t n_adj, h, l, t; - mpd_uint_t n1_neg; - - /* n1_neg = if lo >= 2**63 then MPD_UINT_MAX else 0 */ - n1_neg = (lo & (1ULL<<63)) ? MPD_UINT_MAX : 0; - /* n_adj = if lo >= 2**63 then lo+MPD_RADIX else lo */ - n_adj = lo + (n1_neg & MPD_RADIX); - - /* (h, l) = if lo >= 2**63 then m'*(hi+1) else m'*hi */ - _mpd_mul_words(&h, &l, mprime_rdx, hi-n1_neg); - l = l + n_adj; - if (l < n_adj) h++; - t = h + hi; - /* At this point t == qest, with q == qest or q == qest+1: - * 1) 0 <= 2**64*hi + lo - qest*MPD_RADIX < 2*MPD_RADIX - */ - - /* t = 2**64-1 - qest = 2**64 - (qest+1) */ - t = MPD_UINT_MAX - t; - - /* (h, l) = 2**64*MPD_RADIX - (qest+1)*MPD_RADIX */ - _mpd_mul_words(&h, &l, t, MPD_RADIX); - l = l + lo; - if (l < lo) h++; - h += hi; - h -= MPD_RADIX; - /* (h, l) = 2**64*hi + lo - (qest+1)*MPD_RADIX (mod 2**128) - * Case q == qest+1: - * a) h == 0, l == r - * b) q := h - t == qest+1 - * c) r := l - * Case q == qest: - * a) h == MPD_UINT_MAX, l == 2**64-(MPD_RADIX-r) - * b) q := h - t == qest - * c) r := l + MPD_RADIX = r - */ - - *q = (h - t); - *r = l + (MPD_RADIX & h); -} -#else -static inline void -_mpd_div_words_r(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t hi, mpd_uint_t lo) -{ - _mpd_div_words(q, r, hi, lo, MPD_RADIX); -} -#endif - - -/* Multiply two single base MPD_RADIX words, store result in array w[2]. */ -static inline void -_mpd_singlemul(mpd_uint_t w[2], mpd_uint_t u, mpd_uint_t v) -{ - mpd_uint_t hi, lo; - - _mpd_mul_words(&hi, &lo, u, v); - _mpd_div_words_r(&w[1], &w[0], hi, lo); -} - -/* Multiply u (len 2) and v (len m, 1 <= m <= 2). */ -static inline void -_mpd_mul_2_le2(mpd_uint_t w[4], mpd_uint_t u[2], mpd_uint_t v[2], mpd_ssize_t m) -{ - mpd_uint_t hi, lo; - - _mpd_mul_words(&hi, &lo, u[0], v[0]); - _mpd_div_words_r(&w[1], &w[0], hi, lo); - - _mpd_mul_words(&hi, &lo, u[1], v[0]); - lo = w[1] + lo; - if (lo < w[1]) hi++; - _mpd_div_words_r(&w[2], &w[1], hi, lo); - if (m == 1) return; - - _mpd_mul_words(&hi, &lo, u[0], v[1]); - lo = w[1] + lo; - if (lo < w[1]) hi++; - _mpd_div_words_r(&w[3], &w[1], hi, lo); - - _mpd_mul_words(&hi, &lo, u[1], v[1]); - lo = w[2] + lo; - if (lo < w[2]) hi++; - lo = w[3] + lo; - if (lo < w[3]) hi++; - _mpd_div_words_r(&w[3], &w[2], hi, lo); -} - - -/* - * Test if all words from data[len-1] to data[0] are zero. If len is 0, nothing - * is tested and the coefficient is regarded as "all zero". - */ -static inline int -_mpd_isallzero(const mpd_uint_t *data, mpd_ssize_t len) -{ - while (--len >= 0) { - if (data[len] != 0) return 0; - } - return 1; -} - -/* - * Test if all full words from data[len-1] to data[0] are MPD_RADIX-1 - * (all nines). Return true if len == 0. - */ -static inline int -_mpd_isallnine(const mpd_uint_t *data, mpd_ssize_t len) -{ - while (--len >= 0) { - if (data[len] != MPD_RADIX-1) return 0; - } - return 1; -} - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_BASEARITH_H_ */ diff --git a/Modules/_decimal/libmpdec/bench.c b/Modules/_decimal/libmpdec/bench.c deleted file mode 100644 index 09138f4ce9c03a9..000000000000000 --- a/Modules/_decimal/libmpdec/bench.c +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <time.h> - - -static void -err_exit(const char *msg) -{ - fprintf(stderr, "%s\n", msg); - exit(1); -} - -static mpd_t * -new_mpd(void) -{ - mpd_t *x = mpd_qnew(); - if (x == NULL) { - err_exit("out of memory"); - } - - return x; -} - -/* Nonsense version of escape-time algorithm for calculating a mandelbrot - * set. Just for benchmarking. */ -static void -color_point(mpd_t *x0, mpd_t *y0, long maxiter, mpd_context_t *ctx) -{ - mpd_t *x, *y, *sq_x, *sq_y; - mpd_t *two; - - x = new_mpd(); - y = new_mpd(); - mpd_set_u32(x, 0, ctx); - mpd_set_u32(y, 0, ctx); - - sq_x = new_mpd(); - sq_y = new_mpd(); - mpd_set_u32(sq_x, 0, ctx); - mpd_set_u32(sq_y, 0, ctx); - - two = new_mpd(); - mpd_set_u32(two, 2, ctx); - - for (long i = 0; i < maxiter; i++) { - mpd_mul(y, x, y, ctx); - mpd_mul(y, y, two, ctx); - mpd_add(y, y, y0, ctx); - - mpd_sub(x, sq_x, sq_y, ctx); - mpd_add(x, x, x0, ctx); - - mpd_mul(sq_x, x, x, ctx); - mpd_mul(sq_y, y, y, ctx); - } - - mpd_copy(x0, x, ctx); - - mpd_del(two); - mpd_del(sq_y); - mpd_del(sq_x); - mpd_del(y); - mpd_del(x); -} - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *x0, *y0; - uint32_t prec = 19; - long iter = 10000000; - clock_t start_clock, end_clock; - - if (argc != 3) { - err_exit("usage: bench prec iter\n"); - } - prec = strtoul(argv[1], NULL, 10); - iter = strtol(argv[2], NULL, 10); - - mpd_init(&ctx, prec); - /* no more MPD_MINALLOC changes after here */ - - x0 = new_mpd(); - y0 = new_mpd(); - mpd_set_string(x0, "0.222", &ctx); - mpd_set_string(y0, "0.333", &ctx); - if (ctx.status & MPD_Errors) { - mpd_del(y0); - mpd_del(x0); - err_exit("unexpected error during conversion"); - } - - start_clock = clock(); - color_point(x0, y0, iter, &ctx); - end_clock = clock(); - - mpd_print(x0); - fprintf(stderr, "time: %f\n\n", (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - mpd_del(y0); - mpd_del(x0); - - return 0; -} diff --git a/Modules/_decimal/libmpdec/bench_full.c b/Modules/_decimal/libmpdec/bench_full.c deleted file mode 100644 index 6ab73917e1c32c1..000000000000000 --- a/Modules/_decimal/libmpdec/bench_full.c +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <time.h> - - -static void -err_exit(const char *msg) -{ - fprintf(stderr, "%s\n", msg); - exit(1); -} - -static mpd_t * -new_mpd(void) -{ - mpd_t *x = mpd_qnew(); - if (x == NULL) { - err_exit("out of memory"); - } - - return x; -} - -/* - * Example from: http://en.wikipedia.org/wiki/Mandelbrot_set - * - * Escape time algorithm for drawing the set: - * - * Point x0, y0 is deemed to be in the Mandelbrot set if the return - * value is maxiter. Lower return values indicate how quickly points - * escaped and can be used for coloring. - */ -static int -color_point(const mpd_t *x0, const mpd_t *y0, const long maxiter, mpd_context_t *ctx) -{ - mpd_t *x, *y, *sq_x, *sq_y; - mpd_t *two, *four, *c; - long i; - - x = new_mpd(); - y = new_mpd(); - mpd_set_u32(x, 0, ctx); - mpd_set_u32(y, 0, ctx); - - sq_x = new_mpd(); - sq_y = new_mpd(); - mpd_set_u32(sq_x, 0, ctx); - mpd_set_u32(sq_y, 0, ctx); - - two = new_mpd(); - four = new_mpd(); - mpd_set_u32(two, 2, ctx); - mpd_set_u32(four, 4, ctx); - - c = new_mpd(); - mpd_set_u32(c, 0, ctx); - - for (i = 0; i < maxiter && mpd_cmp(c, four, ctx) <= 0; i++) { - mpd_mul(y, x, y, ctx); - mpd_mul(y, y, two, ctx); - mpd_add(y, y, y0, ctx); - - mpd_sub(x, sq_x, sq_y, ctx); - mpd_add(x, x, x0, ctx); - - mpd_mul(sq_x, x, x, ctx); - mpd_mul(sq_y, y, y, ctx); - mpd_add(c, sq_x, sq_y, ctx); - } - - mpd_del(c); - mpd_del(four); - mpd_del(two); - mpd_del(sq_y); - mpd_del(sq_x); - mpd_del(y); - mpd_del(x); - - return i; -} - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *x0, *y0; - mpd_t *sqrt_2, *xstep, *ystep; - mpd_ssize_t prec = 19; - - long iter = 1000; - int points[40][80]; - int i, j; - clock_t start_clock, end_clock; - - - if (argc != 3) { - fprintf(stderr, "usage: ./bench prec iter\n"); - exit(1); - } - prec = strtoll(argv[1], NULL, 10); - iter = strtol(argv[2], NULL, 10); - - mpd_init(&ctx, prec); - /* no more MPD_MINALLOC changes after here */ - - sqrt_2 = new_mpd(); - xstep = new_mpd(); - ystep = new_mpd(); - x0 = new_mpd(); - y0 = new_mpd(); - - mpd_set_u32(sqrt_2, 2, &ctx); - mpd_sqrt(sqrt_2, sqrt_2, &ctx); - mpd_div_u32(xstep, sqrt_2, 40, &ctx); - mpd_div_u32(ystep, sqrt_2, 20, &ctx); - - start_clock = clock(); - mpd_copy(y0, sqrt_2, &ctx); - for (i = 0; i < 40; i++) { - mpd_copy(x0, sqrt_2, &ctx); - mpd_set_negative(x0); - for (j = 0; j < 80; j++) { - points[i][j] = color_point(x0, y0, iter, &ctx); - mpd_add(x0, x0, xstep, &ctx); - } - mpd_sub(y0, y0, ystep, &ctx); - } - end_clock = clock(); - -#ifdef BENCH_VERBOSE - for (i = 0; i < 40; i++) { - for (j = 0; j < 80; j++) { - if (points[i][j] == iter) { - putchar('*'); - } - else if (points[i][j] >= 10) { - putchar('+'); - } - else if (points[i][j] >= 5) { - putchar('.'); - } - else { - putchar(' '); - } - } - putchar('\n'); - } - putchar('\n'); -#else - (void)points; /* suppress gcc warning */ -#endif - - printf("time: %f\n\n", (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - mpd_del(y0); - mpd_del(x0); - mpd_del(ystep); - mpd_del(xstep); - mpd_del(sqrt_2); - - return 0; -} diff --git a/Modules/_decimal/libmpdec/bits.h b/Modules/_decimal/libmpdec/bits.h deleted file mode 100644 index aa9c3e77980c031..000000000000000 --- a/Modules/_decimal/libmpdec/bits.h +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_BITS_H_ -#define LIBMPDEC_BITS_H_ - - -#include "mpdecimal.h" - - -/* Check if n is a power of 2. */ -static inline int -ispower2(mpd_size_t n) -{ - return n != 0 && (n & (n-1)) == 0; -} - -#if defined(ANSI) -/* - * Return the most significant bit position of n from 0 to 31 (63). - * Assumptions: n != 0. - */ -static inline int -mpd_bsr(mpd_size_t n) -{ - int pos = 0; - mpd_size_t tmp; - -#ifdef CONFIG_64 - tmp = n >> 32; - if (tmp != 0) { n = tmp; pos += 32; } -#endif - tmp = n >> 16; - if (tmp != 0) { n = tmp; pos += 16; } - tmp = n >> 8; - if (tmp != 0) { n = tmp; pos += 8; } - tmp = n >> 4; - if (tmp != 0) { n = tmp; pos += 4; } - tmp = n >> 2; - if (tmp != 0) { n = tmp; pos += 2; } - tmp = n >> 1; - if (tmp != 0) { n = tmp; pos += 1; } - - return pos + (int)n - 1; -} - -/* - * Return the least significant bit position of n from 0 to 31 (63). - * Assumptions: n != 0. - */ -static inline int -mpd_bsf(mpd_size_t n) -{ - int pos; - -#ifdef CONFIG_64 - pos = 63; - if (n & 0x00000000FFFFFFFFULL) { pos -= 32; } else { n >>= 32; } - if (n & 0x000000000000FFFFULL) { pos -= 16; } else { n >>= 16; } - if (n & 0x00000000000000FFULL) { pos -= 8; } else { n >>= 8; } - if (n & 0x000000000000000FULL) { pos -= 4; } else { n >>= 4; } - if (n & 0x0000000000000003ULL) { pos -= 2; } else { n >>= 2; } - if (n & 0x0000000000000001ULL) { pos -= 1; } -#else - pos = 31; - if (n & 0x000000000000FFFFUL) { pos -= 16; } else { n >>= 16; } - if (n & 0x00000000000000FFUL) { pos -= 8; } else { n >>= 8; } - if (n & 0x000000000000000FUL) { pos -= 4; } else { n >>= 4; } - if (n & 0x0000000000000003UL) { pos -= 2; } else { n >>= 2; } - if (n & 0x0000000000000001UL) { pos -= 1; } -#endif - return pos; -} -/* END ANSI */ - -#elif defined(ASM) -/* - * Bit scan reverse. Assumptions: a != 0. - */ -static inline int -mpd_bsr(mpd_size_t a) -{ - mpd_size_t retval; - - __asm__ ( -#ifdef CONFIG_64 - "bsrq %1, %0\n\t" -#else - "bsr %1, %0\n\t" -#endif - :"=r" (retval) - :"r" (a) - :"cc" - ); - - return (int)retval; -} - -/* - * Bit scan forward. Assumptions: a != 0. - */ -static inline int -mpd_bsf(mpd_size_t a) -{ - mpd_size_t retval; - - __asm__ ( -#ifdef CONFIG_64 - "bsfq %1, %0\n\t" -#else - "bsf %1, %0\n\t" -#endif - :"=r" (retval) - :"r" (a) - :"cc" - ); - - return (int)retval; -} -/* END ASM */ - -#elif defined(MASM) -#include <intrin.h> -/* - * Bit scan reverse. Assumptions: a != 0. - */ -static inline int __cdecl -mpd_bsr(mpd_size_t a) -{ - unsigned long retval; - -#ifdef CONFIG_64 - _BitScanReverse64(&retval, a); -#else - _BitScanReverse(&retval, a); -#endif - - return (int)retval; -} - -/* - * Bit scan forward. Assumptions: a != 0. - */ -static inline int __cdecl -mpd_bsf(mpd_size_t a) -{ - unsigned long retval; - -#ifdef CONFIG_64 - _BitScanForward64(&retval, a); -#else - _BitScanForward(&retval, a); -#endif - - return (int)retval; -} -/* END MASM (_MSC_VER) */ -#else - #error "missing preprocessor definitions" -#endif /* BSR/BSF */ - - -#endif /* LIBMPDEC_BITS_H_ */ diff --git a/Modules/_decimal/libmpdec/constants.c b/Modules/_decimal/libmpdec/constants.c deleted file mode 100644 index ed074fa81c6d2e5..000000000000000 --- a/Modules/_decimal/libmpdec/constants.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" -#include "basearith.h" -#include "constants.h" - - -#if defined(CONFIG_64) - - /* number-theory.c */ - const mpd_uint_t mpd_moduli[3] = { - 18446744069414584321ULL, 18446744056529682433ULL, 18446742974197923841ULL - }; - const mpd_uint_t mpd_roots[3] = {7ULL, 10ULL, 19ULL}; - - /* crt.c */ - const mpd_uint_t INV_P1_MOD_P2 = 18446744055098026669ULL; - const mpd_uint_t INV_P1P2_MOD_P3 = 287064143708160ULL; - const mpd_uint_t LH_P1P2 = 18446744052234715137ULL; /* (P1*P2) % 2^64 */ - const mpd_uint_t UH_P1P2 = 18446744052234715141ULL; /* (P1*P2) / 2^64 */ - - /* transpose.c */ - const mpd_size_t mpd_bits[64] = { - 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, - 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, - 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, 1073741824, - 2147483648ULL, 4294967296ULL, 8589934592ULL, 17179869184ULL, 34359738368ULL, - 68719476736ULL, 137438953472ULL, 274877906944ULL, 549755813888ULL, - 1099511627776ULL, 2199023255552ULL, 4398046511104, 8796093022208ULL, - 17592186044416ULL, 35184372088832ULL, 70368744177664ULL, 140737488355328ULL, - 281474976710656ULL, 562949953421312ULL, 1125899906842624ULL, - 2251799813685248ULL, 4503599627370496ULL, 9007199254740992ULL, - 18014398509481984ULL, 36028797018963968ULL, 72057594037927936ULL, - 144115188075855872ULL, 288230376151711744ULL, 576460752303423488ULL, - 1152921504606846976ULL, 2305843009213693952ULL, 4611686018427387904ULL, - 9223372036854775808ULL - }; - - /* mpdecimal.c */ - const mpd_uint_t mpd_pow10[MPD_RDIGITS+1] = { - 1,10,100,1000,10000,100000,1000000,10000000,100000000,1000000000, - 10000000000ULL,100000000000ULL,1000000000000ULL,10000000000000ULL, - 100000000000000ULL,1000000000000000ULL,10000000000000000ULL, - 100000000000000000ULL,1000000000000000000ULL,10000000000000000000ULL - }; - - /* magic number for constant division by MPD_RADIX */ - const mpd_uint_t mprime_rdx = 15581492618384294730ULL; - -#elif defined(CONFIG_32) - - /* number-theory.c */ - const mpd_uint_t mpd_moduli[3] = {2113929217UL, 2013265921UL, 1811939329UL}; - const mpd_uint_t mpd_roots[3] = {5UL, 31UL, 13UL}; - - /* PentiumPro modular multiplication: These constants have to be loaded as - * 80 bit long doubles, which are not supported by certain compilers. */ - const uint32_t mpd_invmoduli[3][3] = { - {4293885170U, 2181570688U, 16352U}, /* ((long double) 1 / 2113929217UL) */ - {1698898177U, 2290649223U, 16352U}, /* ((long double) 1 / 2013265921UL) */ - {2716021846U, 2545165803U, 16352U} /* ((long double) 1 / 1811939329UL) */ - }; - - const float MPD_TWO63 = 9223372036854775808.0; /* 2^63 */ - - /* crt.c */ - const mpd_uint_t INV_P1_MOD_P2 = 2013265901UL; - const mpd_uint_t INV_P1P2_MOD_P3 = 54UL; - const mpd_uint_t LH_P1P2 = 4127195137UL; /* (P1*P2) % 2^32 */ - const mpd_uint_t UH_P1P2 = 990904320UL; /* (P1*P2) / 2^32 */ - - /* transpose.c */ - const mpd_size_t mpd_bits[32] = { - 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, - 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, - 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, 1073741824, - 2147483648UL - }; - - /* mpdecimal.c */ - const mpd_uint_t mpd_pow10[MPD_RDIGITS+1] = { - 1,10,100,1000,10000,100000,1000000,10000000,100000000,1000000000 - }; - -#else - #error "CONFIG_64 or CONFIG_32 must be defined." -#endif - -const char * const mpd_round_string[MPD_ROUND_GUARD] = { - "ROUND_UP", /* round away from 0 */ - "ROUND_DOWN", /* round toward 0 (truncate) */ - "ROUND_CEILING", /* round toward +infinity */ - "ROUND_FLOOR", /* round toward -infinity */ - "ROUND_HALF_UP", /* 0.5 is rounded up */ - "ROUND_HALF_DOWN", /* 0.5 is rounded down */ - "ROUND_HALF_EVEN", /* 0.5 is rounded to even */ - "ROUND_05UP", /* round zero or five away from 0 */ - "ROUND_TRUNC", /* truncate, but set infinity */ -}; - -const char * const mpd_clamp_string[MPD_CLAMP_GUARD] = { - "CLAMP_DEFAULT", - "CLAMP_IEEE_754" -}; diff --git a/Modules/_decimal/libmpdec/constants.h b/Modules/_decimal/libmpdec/constants.h deleted file mode 100644 index 7c1db839c20ba28..000000000000000 --- a/Modules/_decimal/libmpdec/constants.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_CONSTANTS_H_ -#define LIBMPDEC_CONSTANTS_H_ - - -#include "mpdecimal.h" - -#include <stdint.h> - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -/* choice of optimized functions */ -#if defined(CONFIG_64) -/* x64 */ - #define MULMOD(a, b) x64_mulmod(a, b, umod) - #define MULMOD2C(a0, a1, w) x64_mulmod2c(a0, a1, w, umod) - #define MULMOD2(a0, b0, a1, b1) x64_mulmod2(a0, b0, a1, b1, umod) - #define POWMOD(base, exp) x64_powmod(base, exp, umod) - #define SETMODULUS(modnum) std_setmodulus(modnum, &umod) - #define SIZE3_NTT(x0, x1, x2, w3table) std_size3_ntt(x0, x1, x2, w3table, umod) -#elif defined(PPRO) -/* PentiumPro (or later) gcc inline asm */ - #define MULMOD(a, b) ppro_mulmod(a, b, &dmod, dinvmod) - #define MULMOD2C(a0, a1, w) ppro_mulmod2c(a0, a1, w, &dmod, dinvmod) - #define MULMOD2(a0, b0, a1, b1) ppro_mulmod2(a0, b0, a1, b1, &dmod, dinvmod) - #define POWMOD(base, exp) ppro_powmod(base, exp, &dmod, dinvmod) - #define SETMODULUS(modnum) ppro_setmodulus(modnum, &umod, &dmod, dinvmod) - #define SIZE3_NTT(x0, x1, x2, w3table) ppro_size3_ntt(x0, x1, x2, w3table, umod, &dmod, dinvmod) -#else - /* ANSI C99 */ - #define MULMOD(a, b) std_mulmod(a, b, umod) - #define MULMOD2C(a0, a1, w) std_mulmod2c(a0, a1, w, umod) - #define MULMOD2(a0, b0, a1, b1) std_mulmod2(a0, b0, a1, b1, umod) - #define POWMOD(base, exp) std_powmod(base, exp, umod) - #define SETMODULUS(modnum) std_setmodulus(modnum, &umod) - #define SIZE3_NTT(x0, x1, x2, w3table) std_size3_ntt(x0, x1, x2, w3table, umod) -#endif - -/* PentiumPro (or later) gcc inline asm */ -extern const float MPD_TWO63; -extern const uint32_t mpd_invmoduli[3][3]; - -enum {P1, P2, P3}; - -extern const mpd_uint_t mpd_moduli[]; -extern const mpd_uint_t mpd_roots[]; -extern const mpd_size_t mpd_bits[]; -extern const mpd_uint_t mpd_pow10[]; - -extern const mpd_uint_t INV_P1_MOD_P2; -extern const mpd_uint_t INV_P1P2_MOD_P3; -extern const mpd_uint_t LH_P1P2; -extern const mpd_uint_t UH_P1P2; - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_CONSTANTS_H_ */ diff --git a/Modules/_decimal/libmpdec/context.c b/Modules/_decimal/libmpdec/context.c deleted file mode 100644 index 172794b67d80094..000000000000000 --- a/Modules/_decimal/libmpdec/context.c +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <signal.h> -#include <stdio.h> -#include <string.h> - - -void -mpd_dflt_traphandler(mpd_context_t *ctx) -{ - (void)ctx; - raise(SIGFPE); -} - -void (* mpd_traphandler)(mpd_context_t *) = mpd_dflt_traphandler; - - -/* Set guaranteed minimum number of coefficient words. The function may - be used once at program start. Setting MPD_MINALLOC to out-of-bounds - values is a catastrophic error, so in that case the function exits rather - than relying on the user to check a return value. */ -void -mpd_setminalloc(mpd_ssize_t n) -{ - static int minalloc_is_set = 0; - - if (minalloc_is_set) { - mpd_err_warn("mpd_setminalloc: ignoring request to set " - "MPD_MINALLOC a second time\n"); - return; - } - if (n < MPD_MINALLOC_MIN || n > MPD_MINALLOC_MAX) { - mpd_err_fatal("illegal value for MPD_MINALLOC"); /* GCOV_NOT_REACHED */ - } - MPD_MINALLOC = n; - minalloc_is_set = 1; -} - -void -mpd_init(mpd_context_t *ctx, mpd_ssize_t prec) -{ - mpd_ssize_t ideal_minalloc; - - mpd_defaultcontext(ctx); - - if (!mpd_qsetprec(ctx, prec)) { - mpd_addstatus_raise(ctx, MPD_Invalid_context); - return; - } - - ideal_minalloc = 2 * ((prec+MPD_RDIGITS-1) / MPD_RDIGITS); - if (ideal_minalloc < MPD_MINALLOC_MIN) ideal_minalloc = MPD_MINALLOC_MIN; - if (ideal_minalloc > MPD_MINALLOC_MAX) ideal_minalloc = MPD_MINALLOC_MAX; - - mpd_setminalloc(ideal_minalloc); -} - -void -mpd_maxcontext(mpd_context_t *ctx) -{ - ctx->prec=MPD_MAX_PREC; - ctx->emax=MPD_MAX_EMAX; - ctx->emin=MPD_MIN_EMIN; - ctx->round=MPD_ROUND_HALF_EVEN; - ctx->traps=MPD_Traps; - ctx->status=0; - ctx->newtrap=0; - ctx->clamp=0; - ctx->allcr=1; -} - -void -mpd_defaultcontext(mpd_context_t *ctx) -{ - ctx->prec=2*MPD_RDIGITS; - ctx->emax=MPD_MAX_EMAX; - ctx->emin=MPD_MIN_EMIN; - ctx->round=MPD_ROUND_HALF_UP; - ctx->traps=MPD_Traps; - ctx->status=0; - ctx->newtrap=0; - ctx->clamp=0; - ctx->allcr=1; -} - -void -mpd_basiccontext(mpd_context_t *ctx) -{ - ctx->prec=9; - ctx->emax=MPD_MAX_EMAX; - ctx->emin=MPD_MIN_EMIN; - ctx->round=MPD_ROUND_HALF_UP; - ctx->traps=MPD_Traps|MPD_Clamped; - ctx->status=0; - ctx->newtrap=0; - ctx->clamp=0; - ctx->allcr=1; -} - -int -mpd_ieee_context(mpd_context_t *ctx, int bits) -{ - if (bits <= 0 || bits > MPD_IEEE_CONTEXT_MAX_BITS || bits % 32) { - return -1; - } - - ctx->prec = 9 * (bits/32) - 2; - ctx->emax = 3 * ((mpd_ssize_t)1<<(bits/16+3)); - ctx->emin = 1 - ctx->emax; - ctx->round=MPD_ROUND_HALF_EVEN; - ctx->traps=0; - ctx->status=0; - ctx->newtrap=0; - ctx->clamp=1; - ctx->allcr=1; - - return 0; -} - -mpd_ssize_t -mpd_getprec(const mpd_context_t *ctx) -{ - return ctx->prec; -} - -mpd_ssize_t -mpd_getemax(const mpd_context_t *ctx) -{ - return ctx->emax; -} - -mpd_ssize_t -mpd_getemin(const mpd_context_t *ctx) -{ - return ctx->emin; -} - -int -mpd_getround(const mpd_context_t *ctx) -{ - return ctx->round; -} - -uint32_t -mpd_gettraps(const mpd_context_t *ctx) -{ - return ctx->traps; -} - -uint32_t -mpd_getstatus(const mpd_context_t *ctx) -{ - return ctx->status; -} - -int -mpd_getclamp(const mpd_context_t *ctx) -{ - return ctx->clamp; -} - -int -mpd_getcr(const mpd_context_t *ctx) -{ - return ctx->allcr; -} - - -int -mpd_qsetprec(mpd_context_t *ctx, mpd_ssize_t prec) -{ - if (prec <= 0 || prec > MPD_MAX_PREC) { - return 0; - } - ctx->prec = prec; - return 1; -} - -int -mpd_qsetemax(mpd_context_t *ctx, mpd_ssize_t emax) -{ - if (emax < 0 || emax > MPD_MAX_EMAX) { - return 0; - } - ctx->emax = emax; - return 1; -} - -int -mpd_qsetemin(mpd_context_t *ctx, mpd_ssize_t emin) -{ - if (emin > 0 || emin < MPD_MIN_EMIN) { - return 0; - } - ctx->emin = emin; - return 1; -} - -int -mpd_qsetround(mpd_context_t *ctx, int round) -{ - if (!(0 <= round && round < MPD_ROUND_GUARD)) { - return 0; - } - ctx->round = round; - return 1; -} - -int -mpd_qsettraps(mpd_context_t *ctx, uint32_t flags) -{ - if (flags > MPD_Max_status) { - return 0; - } - ctx->traps = flags; - return 1; -} - -int -mpd_qsetstatus(mpd_context_t *ctx, uint32_t flags) -{ - if (flags > MPD_Max_status) { - return 0; - } - ctx->status = flags; - return 1; -} - -int -mpd_qsetclamp(mpd_context_t *ctx, int c) -{ - if (c != 0 && c != 1) { - return 0; - } - ctx->clamp = c; - return 1; -} - -int -mpd_qsetcr(mpd_context_t *ctx, int c) -{ - if (c != 0 && c != 1) { - return 0; - } - ctx->allcr = c; - return 1; -} - - -void -mpd_addstatus_raise(mpd_context_t *ctx, uint32_t flags) -{ - ctx->status |= flags; - if (flags&ctx->traps) { - ctx->newtrap = (flags&ctx->traps); - mpd_traphandler(ctx); - } -} diff --git a/Modules/_decimal/libmpdec/convolute.c b/Modules/_decimal/libmpdec/convolute.c deleted file mode 100644 index 4bc8e8b5fd32f48..000000000000000 --- a/Modules/_decimal/libmpdec/convolute.c +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" -#include "bits.h" -#include "constants.h" -#include "convolute.h" -#include "fnt.h" -#include "fourstep.h" -#include "numbertheory.h" -#include "sixstep.h" -#include "umodarith.h" - - -/* Bignum: Fast convolution using the Number Theoretic Transform. Used for - the multiplication of very large coefficients. */ - - -/* Convolute the data in c1 and c2. Result is in c1. */ -int -fnt_convolute(mpd_uint_t *c1, mpd_uint_t *c2, mpd_size_t n, int modnum) -{ - int (*fnt)(mpd_uint_t *, mpd_size_t, int); - int (*inv_fnt)(mpd_uint_t *, mpd_size_t, int); -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_uint_t n_inv, umod; - mpd_size_t i; - - - SETMODULUS(modnum); - n_inv = POWMOD(n, (umod-2)); - - if (ispower2(n)) { - if (n > SIX_STEP_THRESHOLD) { - fnt = six_step_fnt; - inv_fnt = inv_six_step_fnt; - } - else { - fnt = std_fnt; - inv_fnt = std_inv_fnt; - } - } - else { - fnt = four_step_fnt; - inv_fnt = inv_four_step_fnt; - } - - if (!fnt(c1, n, modnum)) { - return 0; - } - if (!fnt(c2, n, modnum)) { - return 0; - } - for (i = 0; i < n-1; i += 2) { - mpd_uint_t x0 = c1[i]; - mpd_uint_t y0 = c2[i]; - mpd_uint_t x1 = c1[i+1]; - mpd_uint_t y1 = c2[i+1]; - MULMOD2(&x0, y0, &x1, y1); - c1[i] = x0; - c1[i+1] = x1; - } - - if (!inv_fnt(c1, n, modnum)) { - return 0; - } - for (i = 0; i < n-3; i += 4) { - mpd_uint_t x0 = c1[i]; - mpd_uint_t x1 = c1[i+1]; - mpd_uint_t x2 = c1[i+2]; - mpd_uint_t x3 = c1[i+3]; - MULMOD2C(&x0, &x1, n_inv); - MULMOD2C(&x2, &x3, n_inv); - c1[i] = x0; - c1[i+1] = x1; - c1[i+2] = x2; - c1[i+3] = x3; - } - - return 1; -} - -/* Autoconvolute the data in c1. Result is in c1. */ -int -fnt_autoconvolute(mpd_uint_t *c1, mpd_size_t n, int modnum) -{ - int (*fnt)(mpd_uint_t *, mpd_size_t, int); - int (*inv_fnt)(mpd_uint_t *, mpd_size_t, int); -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_uint_t n_inv, umod; - mpd_size_t i; - - - SETMODULUS(modnum); - n_inv = POWMOD(n, (umod-2)); - - if (ispower2(n)) { - if (n > SIX_STEP_THRESHOLD) { - fnt = six_step_fnt; - inv_fnt = inv_six_step_fnt; - } - else { - fnt = std_fnt; - inv_fnt = std_inv_fnt; - } - } - else { - fnt = four_step_fnt; - inv_fnt = inv_four_step_fnt; - } - - if (!fnt(c1, n, modnum)) { - return 0; - } - for (i = 0; i < n-1; i += 2) { - mpd_uint_t x0 = c1[i]; - mpd_uint_t x1 = c1[i+1]; - MULMOD2(&x0, x0, &x1, x1); - c1[i] = x0; - c1[i+1] = x1; - } - - if (!inv_fnt(c1, n, modnum)) { - return 0; - } - for (i = 0; i < n-3; i += 4) { - mpd_uint_t x0 = c1[i]; - mpd_uint_t x1 = c1[i+1]; - mpd_uint_t x2 = c1[i+2]; - mpd_uint_t x3 = c1[i+3]; - MULMOD2C(&x0, &x1, n_inv); - MULMOD2C(&x2, &x3, n_inv); - c1[i] = x0; - c1[i+1] = x1; - c1[i+2] = x2; - c1[i+3] = x3; - } - - return 1; -} diff --git a/Modules/_decimal/libmpdec/convolute.h b/Modules/_decimal/libmpdec/convolute.h deleted file mode 100644 index 62edb3e45739cb0..000000000000000 --- a/Modules/_decimal/libmpdec/convolute.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_CONVOLUTE_H_ -#define LIBMPDEC_CONVOLUTE_H_ - - -#include "mpdecimal.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -#define SIX_STEP_THRESHOLD 4096 - -int fnt_convolute(mpd_uint_t *c1, mpd_uint_t *c2, mpd_size_t n, int modnum); -int fnt_autoconvolute(mpd_uint_t *c1, mpd_size_t n, int modnum); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_CONVOLUTE_H_ */ diff --git a/Modules/_decimal/libmpdec/crt.c b/Modules/_decimal/libmpdec/crt.c deleted file mode 100644 index babcce41bf67cdd..000000000000000 --- a/Modules/_decimal/libmpdec/crt.c +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> - -#include "constants.h" -#include "crt.h" -#include "numbertheory.h" -#include "typearith.h" -#include "umodarith.h" - - -/* Bignum: Chinese Remainder Theorem, extends the maximum transform length. */ - - -/* Multiply P1P2 by v, store result in w. */ -static inline void -_crt_mulP1P2_3(mpd_uint_t w[3], mpd_uint_t v) -{ - mpd_uint_t hi1, hi2, lo; - - _mpd_mul_words(&hi1, &lo, LH_P1P2, v); - w[0] = lo; - - _mpd_mul_words(&hi2, &lo, UH_P1P2, v); - lo = hi1 + lo; - if (lo < hi1) hi2++; - - w[1] = lo; - w[2] = hi2; -} - -/* Add 3 words from v to w. The result is known to fit in w. */ -static inline void -_crt_add3(mpd_uint_t w[3], mpd_uint_t v[3]) -{ - mpd_uint_t carry; - - w[0] = w[0] + v[0]; - carry = (w[0] < v[0]); - - w[1] = w[1] + v[1]; - if (w[1] < v[1]) w[2]++; - - w[1] = w[1] + carry; - if (w[1] < carry) w[2]++; - - w[2] += v[2]; -} - -/* Divide 3 words in u by v, store result in w, return remainder. */ -static inline mpd_uint_t -_crt_div3(mpd_uint_t *w, const mpd_uint_t *u, mpd_uint_t v) -{ - mpd_uint_t r1 = u[2]; - mpd_uint_t r2; - - if (r1 < v) { - w[2] = 0; - } - else { - _mpd_div_word(&w[2], &r1, u[2], v); /* GCOV_NOT_REACHED */ - } - - _mpd_div_words(&w[1], &r2, r1, u[1], v); - _mpd_div_words(&w[0], &r1, r2, u[0], v); - - return r1; -} - - -/* - * Chinese Remainder Theorem: - * Algorithm from Joerg Arndt, "Matters Computational", - * Chapter 37.4.1 [http://www.jjj.de/fxt/] - * - * See also Knuth, TAOCP, Volume 2, 4.3.2, exercise 7. - */ - -/* - * CRT with carry: x1, x2, x3 contain numbers modulo p1, p2, p3. For each - * triple of members of the arrays, find the unique z modulo p1*p2*p3, with - * zmax = p1*p2*p3 - 1. - * - * In each iteration of the loop, split z into result[i] = z % MPD_RADIX - * and carry = z / MPD_RADIX. Let N be the size of carry[] and cmax the - * maximum carry. - * - * Limits for the 32-bit build: - * - * N = 2**96 - * cmax = 7711435591312380274 - * - * Limits for the 64 bit build: - * - * N = 2**192 - * cmax = 627710135393475385904124401220046371710 - * - * The following statements hold for both versions: - * - * 1) cmax + zmax < N, so the addition does not overflow. - * - * 2) (cmax + zmax) / MPD_RADIX == cmax. - * - * 3) If c <= cmax, then c_next = (c + zmax) / MPD_RADIX <= cmax. - */ -void -crt3(mpd_uint_t *x1, mpd_uint_t *x2, mpd_uint_t *x3, mpd_size_t rsize) -{ - mpd_uint_t p1 = mpd_moduli[P1]; - mpd_uint_t umod; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_uint_t a1, a2, a3; - mpd_uint_t s; - mpd_uint_t z[3], t[3]; - mpd_uint_t carry[3] = {0,0,0}; - mpd_uint_t hi, lo; - mpd_size_t i; - - for (i = 0; i < rsize; i++) { - - a1 = x1[i]; - a2 = x2[i]; - a3 = x3[i]; - - SETMODULUS(P2); - s = ext_submod(a2, a1, umod); - s = MULMOD(s, INV_P1_MOD_P2); - - _mpd_mul_words(&hi, &lo, s, p1); - lo = lo + a1; - if (lo < a1) hi++; - - SETMODULUS(P3); - s = dw_submod(a3, hi, lo, umod); - s = MULMOD(s, INV_P1P2_MOD_P3); - - z[0] = lo; - z[1] = hi; - z[2] = 0; - - _crt_mulP1P2_3(t, s); - _crt_add3(z, t); - _crt_add3(carry, z); - - x1[i] = _crt_div3(carry, carry, MPD_RADIX); - } - - assert(carry[0] == 0 && carry[1] == 0 && carry[2] == 0); -} diff --git a/Modules/_decimal/libmpdec/crt.h b/Modules/_decimal/libmpdec/crt.h deleted file mode 100644 index ed66753c2510ba8..000000000000000 --- a/Modules/_decimal/libmpdec/crt.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_CRT_H_ -#define LIBMPDEC_CRT_H_ - - -#include "mpdecimal.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -void crt3(mpd_uint_t *x1, mpd_uint_t *x2, mpd_uint_t *x3, mpd_size_t rsize); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_CRT_H_ */ diff --git a/Modules/_decimal/libmpdec/difradix2.c b/Modules/_decimal/libmpdec/difradix2.c deleted file mode 100644 index 049ecff65b6eefc..000000000000000 --- a/Modules/_decimal/libmpdec/difradix2.c +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> - -#include "bits.h" -#include "constants.h" -#include "difradix2.h" -#include "numbertheory.h" -#include "umodarith.h" - - -/* Bignum: The actual transform routine (decimation in frequency). */ - - -/* - * Generate index pairs (x, bitreverse(x)) and carry out the permutation. - * n must be a power of two. - * Algorithm due to Brent/Lehmann, see Joerg Arndt, "Matters Computational", - * Chapter 1.14.4. [http://www.jjj.de/fxt/] - */ -static inline void -bitreverse_permute(mpd_uint_t a[], mpd_size_t n) -{ - mpd_size_t x = 0; - mpd_size_t r = 0; - mpd_uint_t t; - - do { /* Invariant: r = bitreverse(x) */ - if (r > x) { - t = a[x]; - a[x] = a[r]; - a[r] = t; - } - /* Flip trailing consecutive 1 bits and the first zero bit - * that absorbs a possible carry. */ - x += 1; - /* Mirror the operation on r: Flip n_trailing_zeros(x)+1 - high bits of r. */ - r ^= (n - (n >> (mpd_bsf(x)+1))); - /* The loop invariant is preserved. */ - } while (x < n); -} - - -/* Fast Number Theoretic Transform, decimation in frequency. */ -void -fnt_dif2(mpd_uint_t a[], mpd_size_t n, struct fnt_params *tparams) -{ - mpd_uint_t *wtable = tparams->wtable; - mpd_uint_t umod; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_uint_t u0, u1, v0, v1; - mpd_uint_t w, w0, w1, wstep; - mpd_size_t m, mhalf; - mpd_size_t j, r; - - - assert(ispower2(n)); - assert(n >= 4); - - SETMODULUS(tparams->modnum); - - /* m == n */ - mhalf = n / 2; - for (j = 0; j < mhalf; j += 2) { - - w0 = wtable[j]; - w1 = wtable[j+1]; - - u0 = a[j]; - v0 = a[j+mhalf]; - - u1 = a[j+1]; - v1 = a[j+1+mhalf]; - - a[j] = addmod(u0, v0, umod); - v0 = submod(u0, v0, umod); - - a[j+1] = addmod(u1, v1, umod); - v1 = submod(u1, v1, umod); - - MULMOD2(&v0, w0, &v1, w1); - - a[j+mhalf] = v0; - a[j+1+mhalf] = v1; - - } - - wstep = 2; - for (m = n/2; m >= 2; m>>=1, wstep<<=1) { - - mhalf = m / 2; - - /* j == 0 */ - for (r = 0; r < n; r += 2*m) { - - u0 = a[r]; - v0 = a[r+mhalf]; - - u1 = a[m+r]; - v1 = a[m+r+mhalf]; - - a[r] = addmod(u0, v0, umod); - v0 = submod(u0, v0, umod); - - a[m+r] = addmod(u1, v1, umod); - v1 = submod(u1, v1, umod); - - a[r+mhalf] = v0; - a[m+r+mhalf] = v1; - } - - for (j = 1; j < mhalf; j++) { - - w = wtable[j*wstep]; - - for (r = 0; r < n; r += 2*m) { - - u0 = a[r+j]; - v0 = a[r+j+mhalf]; - - u1 = a[m+r+j]; - v1 = a[m+r+j+mhalf]; - - a[r+j] = addmod(u0, v0, umod); - v0 = submod(u0, v0, umod); - - a[m+r+j] = addmod(u1, v1, umod); - v1 = submod(u1, v1, umod); - - MULMOD2C(&v0, &v1, w); - - a[r+j+mhalf] = v0; - a[m+r+j+mhalf] = v1; - } - - } - - } - - bitreverse_permute(a, n); -} diff --git a/Modules/_decimal/libmpdec/difradix2.h b/Modules/_decimal/libmpdec/difradix2.h deleted file mode 100644 index cdcbcf9a71043ce..000000000000000 --- a/Modules/_decimal/libmpdec/difradix2.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_DIFRADIX2_H_ -#define LIBMPDEC_DIFRADIX2_H_ - - -#include "mpdecimal.h" -#include "numbertheory.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -void fnt_dif2(mpd_uint_t a[], mpd_size_t n, struct fnt_params *tparams); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_DIFRADIX2_H_ */ diff --git a/Modules/_decimal/libmpdec/examples/README.txt b/Modules/_decimal/libmpdec/examples/README.txt deleted file mode 100644 index 69615b45f9821a6..000000000000000 --- a/Modules/_decimal/libmpdec/examples/README.txt +++ /dev/null @@ -1,8 +0,0 @@ - - -This directory contains a number of examples. In order to compile, run -(for example): - -gcc -Wall -W -O2 -o powmod powmod.c -lmpdec -lm - - diff --git a/Modules/_decimal/libmpdec/examples/compare.c b/Modules/_decimal/libmpdec/examples/compare.c deleted file mode 100644 index 9051773e116dec3..000000000000000 --- a/Modules/_decimal/libmpdec/examples/compare.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <mpdecimal.h> - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *a, *b; - mpd_t *result; - char *rstring; - char status_str[MPD_MAX_FLAG_STRING]; - clock_t start_clock, end_clock; - - if (argc != 3) { - fprintf(stderr, "compare: usage: ./compare x y\n"); - exit(1); - } - - mpd_init(&ctx, 38); - ctx.traps = 0; - - result = mpd_new(&ctx); - a = mpd_new(&ctx); - b = mpd_new(&ctx); - mpd_set_string(a, argv[1], &ctx); - mpd_set_string(b, argv[2], &ctx); - - start_clock = clock(); - mpd_compare(result, a, b, &ctx); - end_clock = clock(); - fprintf(stderr, "time: %f\n\n", - (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - rstring = mpd_to_sci(result, 1); - mpd_snprint_flags(status_str, MPD_MAX_FLAG_STRING, ctx.status); - printf("%s %s\n", rstring, status_str); - - mpd_del(a); - mpd_del(b); - mpd_del(result); - mpd_free(rstring); - - return 0; -} - - diff --git a/Modules/_decimal/libmpdec/examples/div.c b/Modules/_decimal/libmpdec/examples/div.c deleted file mode 100644 index b76037d2a64c67f..000000000000000 --- a/Modules/_decimal/libmpdec/examples/div.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <mpdecimal.h> - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *a, *b; - mpd_t *result; - char *rstring; - char status_str[MPD_MAX_FLAG_STRING]; - clock_t start_clock, end_clock; - - if (argc != 3) { - fprintf(stderr, "div: usage: ./div x y\n"); - exit(1); - } - - mpd_init(&ctx, 38); - ctx.traps = 0; - - result = mpd_new(&ctx); - a = mpd_new(&ctx); - b = mpd_new(&ctx); - mpd_set_string(a, argv[1], &ctx); - mpd_set_string(b, argv[2], &ctx); - - start_clock = clock(); - mpd_div(result, a, b, &ctx); - end_clock = clock(); - fprintf(stderr, "time: %f\n\n", - (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - rstring = mpd_to_sci(result, 1); - mpd_snprint_flags(status_str, MPD_MAX_FLAG_STRING, ctx.status); - printf("%s %s\n", rstring, status_str); - - mpd_del(a); - mpd_del(b); - mpd_del(result); - mpd_free(rstring); - - return 0; -} - - diff --git a/Modules/_decimal/libmpdec/examples/divmod.c b/Modules/_decimal/libmpdec/examples/divmod.c deleted file mode 100644 index 1f2b48306d6d068..000000000000000 --- a/Modules/_decimal/libmpdec/examples/divmod.c +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <mpdecimal.h> - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *a, *b; - mpd_t *q, *r; - char *qs, *rs; - char status_str[MPD_MAX_FLAG_STRING]; - clock_t start_clock, end_clock; - - if (argc != 3) { - fprintf(stderr, "divmod: usage: ./divmod x y\n"); - exit(1); - } - - mpd_init(&ctx, 38); - ctx.traps = 0; - - q = mpd_new(&ctx); - r = mpd_new(&ctx); - a = mpd_new(&ctx); - b = mpd_new(&ctx); - mpd_set_string(a, argv[1], &ctx); - mpd_set_string(b, argv[2], &ctx); - - start_clock = clock(); - mpd_divmod(q, r, a, b, &ctx); - end_clock = clock(); - fprintf(stderr, "time: %f\n\n", - (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - qs = mpd_to_sci(q, 1); - rs = mpd_to_sci(r, 1); - - mpd_snprint_flags(status_str, MPD_MAX_FLAG_STRING, ctx.status); - printf("%s %s %s\n", qs, rs, status_str); - - mpd_del(q); - mpd_del(r); - mpd_del(a); - mpd_del(b); - mpd_free(qs); - mpd_free(rs); - - return 0; -} - - diff --git a/Modules/_decimal/libmpdec/examples/multiply.c b/Modules/_decimal/libmpdec/examples/multiply.c deleted file mode 100644 index 7f2687d15f8273b..000000000000000 --- a/Modules/_decimal/libmpdec/examples/multiply.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <mpdecimal.h> - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *a, *b; - mpd_t *result; - char *rstring; - char status_str[MPD_MAX_FLAG_STRING]; - clock_t start_clock, end_clock; - - if (argc != 3) { - fprintf(stderr, "multiply: usage: ./multiply x y\n"); - exit(1); - } - - mpd_init(&ctx, 38); - ctx.traps = 0; - - result = mpd_new(&ctx); - a = mpd_new(&ctx); - b = mpd_new(&ctx); - mpd_set_string(a, argv[1], &ctx); - mpd_set_string(b, argv[2], &ctx); - - start_clock = clock(); - mpd_mul(result, a, b, &ctx); - end_clock = clock(); - fprintf(stderr, "time: %f\n\n", - (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - rstring = mpd_to_sci(result, 1); - mpd_snprint_flags(status_str, MPD_MAX_FLAG_STRING, ctx.status); - printf("%s %s\n", rstring, status_str); - - mpd_del(a); - mpd_del(b); - mpd_del(result); - mpd_free(rstring); - - return 0; -} - - diff --git a/Modules/_decimal/libmpdec/examples/pow.c b/Modules/_decimal/libmpdec/examples/pow.c deleted file mode 100644 index 628c14342735730..000000000000000 --- a/Modules/_decimal/libmpdec/examples/pow.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <mpdecimal.h> - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *a, *b; - mpd_t *result; - char *rstring; - char status_str[MPD_MAX_FLAG_STRING]; - clock_t start_clock, end_clock; - - if (argc != 3) { - fprintf(stderr, "pow: usage: ./pow x y\n"); - exit(1); - } - - mpd_init(&ctx, 38); - ctx.traps = 0; - - result = mpd_new(&ctx); - a = mpd_new(&ctx); - b = mpd_new(&ctx); - mpd_set_string(a, argv[1], &ctx); - mpd_set_string(b, argv[2], &ctx); - - start_clock = clock(); - mpd_pow(result, a, b, &ctx); - end_clock = clock(); - fprintf(stderr, "time: %f\n\n", - (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - rstring = mpd_to_sci(result, 1); - mpd_snprint_flags(status_str, MPD_MAX_FLAG_STRING, ctx.status); - printf("%s %s\n", rstring, status_str); - - mpd_del(a); - mpd_del(b); - mpd_del(result); - mpd_free(rstring); - - return 0; -} - - diff --git a/Modules/_decimal/libmpdec/examples/powmod.c b/Modules/_decimal/libmpdec/examples/powmod.c deleted file mode 100644 index b422fdbbb955d75..000000000000000 --- a/Modules/_decimal/libmpdec/examples/powmod.c +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <mpdecimal.h> - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *a, *b, *c; - mpd_t *result; - char *rstring; - char status_str[MPD_MAX_FLAG_STRING]; - clock_t start_clock, end_clock; - - if (argc != 4) { - fprintf(stderr, "powmod: usage: ./powmod x y z\n"); - exit(1); - } - - mpd_init(&ctx, 38); - ctx.traps = 0; - - result = mpd_new(&ctx); - a = mpd_new(&ctx); - b = mpd_new(&ctx); - c = mpd_new(&ctx); - mpd_set_string(a, argv[1], &ctx); - mpd_set_string(b, argv[2], &ctx); - mpd_set_string(c, argv[3], &ctx); - - start_clock = clock(); - mpd_powmod(result, a, b, c, &ctx); - end_clock = clock(); - fprintf(stderr, "time: %f\n\n", - (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - rstring = mpd_to_sci(result, 1); - mpd_snprint_flags(status_str, MPD_MAX_FLAG_STRING, ctx.status); - printf("%s %s\n", rstring, status_str); - - mpd_del(a); - mpd_del(b); - mpd_del(c); - mpd_del(result); - mpd_free(rstring); - - return 0; -} - - diff --git a/Modules/_decimal/libmpdec/examples/shift.c b/Modules/_decimal/libmpdec/examples/shift.c deleted file mode 100644 index 6d54e108ca87f04..000000000000000 --- a/Modules/_decimal/libmpdec/examples/shift.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <mpdecimal.h> - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *a, *b; - mpd_t *result; - char *rstring; - char status_str[MPD_MAX_FLAG_STRING]; - clock_t start_clock, end_clock; - - if (argc != 3) { - fprintf(stderr, "shift: usage: ./shift x y\n"); - exit(1); - } - - mpd_init(&ctx, 38); - ctx.traps = 0; - - result = mpd_new(&ctx); - a = mpd_new(&ctx); - b = mpd_new(&ctx); - mpd_set_string(a, argv[1], &ctx); - mpd_set_string(b, argv[2], &ctx); - - start_clock = clock(); - mpd_shift(result, a, b, &ctx); - end_clock = clock(); - fprintf(stderr, "time: %f\n\n", - (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - rstring = mpd_to_sci(result, 1); - mpd_snprint_flags(status_str, MPD_MAX_FLAG_STRING, ctx.status); - printf("%s %s\n", rstring, status_str); - - mpd_del(a); - mpd_del(b); - mpd_del(result); - mpd_free(rstring); - - return 0; -} - - diff --git a/Modules/_decimal/libmpdec/examples/sqrt.c b/Modules/_decimal/libmpdec/examples/sqrt.c deleted file mode 100644 index d8272789b18c236..000000000000000 --- a/Modules/_decimal/libmpdec/examples/sqrt.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include <stdio.h> -#include <stdlib.h> -#include <time.h> -#include <mpdecimal.h> - - -int -main(int argc, char **argv) -{ - mpd_context_t ctx; - mpd_t *a; - mpd_t *result; - char *rstring; - char status_str[MPD_MAX_FLAG_STRING]; - clock_t start_clock, end_clock; - - if (argc != 2) { - fprintf(stderr, "sqrt: usage: ./sqrt x\n"); - exit(1); - } - - mpd_init(&ctx, 38); - ctx.traps = 0; - - result = mpd_new(&ctx); - a = mpd_new(&ctx); - mpd_set_string(a, argv[1], &ctx); - - start_clock = clock(); - mpd_sqrt(result, a, &ctx); - end_clock = clock(); - fprintf(stderr, "time: %f\n\n", - (double)(end_clock-start_clock)/(double)CLOCKS_PER_SEC); - - rstring = mpd_to_sci(result, 1); - mpd_snprint_flags(status_str, MPD_MAX_FLAG_STRING, ctx.status); - printf("%s %s\n", rstring, status_str); - - mpd_del(a); - mpd_del(result); - mpd_free(rstring); - - return 0; -} - - diff --git a/Modules/_decimal/libmpdec/fnt.c b/Modules/_decimal/libmpdec/fnt.c deleted file mode 100644 index 0dbe98fc71c9ea6..000000000000000 --- a/Modules/_decimal/libmpdec/fnt.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> -#include <stdio.h> - -#include "bits.h" -#include "difradix2.h" -#include "fnt.h" -#include "numbertheory.h" - - -/* Bignum: Fast transform for medium-sized coefficients. */ - - -/* forward transform, sign = -1 */ -int -std_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) -{ - struct fnt_params *tparams; - - assert(ispower2(n)); - assert(n >= 4); - assert(n <= 3*MPD_MAXTRANSFORM_2N); - - if ((tparams = _mpd_init_fnt_params(n, -1, modnum)) == NULL) { - return 0; - } - fnt_dif2(a, n, tparams); - - mpd_free(tparams); - return 1; -} - -/* reverse transform, sign = 1 */ -int -std_inv_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) -{ - struct fnt_params *tparams; - - assert(ispower2(n)); - assert(n >= 4); - assert(n <= 3*MPD_MAXTRANSFORM_2N); - - if ((tparams = _mpd_init_fnt_params(n, 1, modnum)) == NULL) { - return 0; - } - fnt_dif2(a, n, tparams); - - mpd_free(tparams); - return 1; -} diff --git a/Modules/_decimal/libmpdec/fnt.h b/Modules/_decimal/libmpdec/fnt.h deleted file mode 100644 index 5222c476a3a4f54..000000000000000 --- a/Modules/_decimal/libmpdec/fnt.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_FNT_H_ -#define LIBMPDEC_FNT_H_ - - -#include "mpdecimal.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -int std_fnt(mpd_uint_t a[], mpd_size_t n, int modnum); -int std_inv_fnt(mpd_uint_t a[], mpd_size_t n, int modnum); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_FNT_H_ */ diff --git a/Modules/_decimal/libmpdec/fourstep.c b/Modules/_decimal/libmpdec/fourstep.c deleted file mode 100644 index fb173ed5a52e463..000000000000000 --- a/Modules/_decimal/libmpdec/fourstep.c +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> - -#include "constants.h" -#include "fourstep.h" -#include "numbertheory.h" -#include "sixstep.h" -#include "umodarith.h" - - -/* Bignum: Cache efficient Matrix Fourier Transform for arrays of the - form 3 * 2**n (See literature/matrix-transform.txt). */ - - -#ifndef PPRO -static inline void -std_size3_ntt(mpd_uint_t *x1, mpd_uint_t *x2, mpd_uint_t *x3, - mpd_uint_t w3table[3], mpd_uint_t umod) -{ - mpd_uint_t r1, r2; - mpd_uint_t w; - mpd_uint_t s, tmp; - - - /* k = 0 -> w = 1 */ - s = *x1; - s = addmod(s, *x2, umod); - s = addmod(s, *x3, umod); - - r1 = s; - - /* k = 1 */ - s = *x1; - - w = w3table[1]; - tmp = MULMOD(*x2, w); - s = addmod(s, tmp, umod); - - w = w3table[2]; - tmp = MULMOD(*x3, w); - s = addmod(s, tmp, umod); - - r2 = s; - - /* k = 2 */ - s = *x1; - - w = w3table[2]; - tmp = MULMOD(*x2, w); - s = addmod(s, tmp, umod); - - w = w3table[1]; - tmp = MULMOD(*x3, w); - s = addmod(s, tmp, umod); - - *x3 = s; - *x2 = r2; - *x1 = r1; -} -#else /* PPRO */ -static inline void -ppro_size3_ntt(mpd_uint_t *x1, mpd_uint_t *x2, mpd_uint_t *x3, mpd_uint_t w3table[3], - mpd_uint_t umod, double *dmod, uint32_t dinvmod[3]) -{ - mpd_uint_t r1, r2; - mpd_uint_t w; - mpd_uint_t s, tmp; - - - /* k = 0 -> w = 1 */ - s = *x1; - s = addmod(s, *x2, umod); - s = addmod(s, *x3, umod); - - r1 = s; - - /* k = 1 */ - s = *x1; - - w = w3table[1]; - tmp = ppro_mulmod(*x2, w, dmod, dinvmod); - s = addmod(s, tmp, umod); - - w = w3table[2]; - tmp = ppro_mulmod(*x3, w, dmod, dinvmod); - s = addmod(s, tmp, umod); - - r2 = s; - - /* k = 2 */ - s = *x1; - - w = w3table[2]; - tmp = ppro_mulmod(*x2, w, dmod, dinvmod); - s = addmod(s, tmp, umod); - - w = w3table[1]; - tmp = ppro_mulmod(*x3, w, dmod, dinvmod); - s = addmod(s, tmp, umod); - - *x3 = s; - *x2 = r2; - *x1 = r1; -} -#endif - - -/* forward transform, sign = -1; transform length = 3 * 2**n */ -int -four_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) -{ - mpd_size_t R = 3; /* number of rows */ - mpd_size_t C = n / 3; /* number of columns */ - mpd_uint_t w3table[3]; - mpd_uint_t kernel, w0, w1, wstep; - mpd_uint_t *s, *p0, *p1, *p2; - mpd_uint_t umod; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_size_t i, k; - - - assert(n >= 48); - assert(n <= 3*MPD_MAXTRANSFORM_2N); - - - /* Length R transform on the columns. */ - SETMODULUS(modnum); - _mpd_init_w3table(w3table, -1, modnum); - for (p0=a, p1=p0+C, p2=p0+2*C; p0<a+C; p0++,p1++,p2++) { - - SIZE3_NTT(p0, p1, p2, w3table); - } - - /* Multiply each matrix element (addressed by i*C+k) by r**(i*k). */ - kernel = _mpd_getkernel(n, -1, modnum); - for (i = 1; i < R; i++) { - w0 = 1; /* r**(i*0): initial value for k=0 */ - w1 = POWMOD(kernel, i); /* r**(i*1): initial value for k=1 */ - wstep = MULMOD(w1, w1); /* r**(2*i) */ - for (k = 0; k < C-1; k += 2) { - mpd_uint_t x0 = a[i*C+k]; - mpd_uint_t x1 = a[i*C+k+1]; - MULMOD2(&x0, w0, &x1, w1); - MULMOD2C(&w0, &w1, wstep); /* r**(i*(k+2)) = r**(i*k) * r**(2*i) */ - a[i*C+k] = x0; - a[i*C+k+1] = x1; - } - } - - /* Length C transform on the rows. */ - for (s = a; s < a+n; s += C) { - if (!six_step_fnt(s, C, modnum)) { - return 0; - } - } - -#if 0 - /* An unordered transform is sufficient for convolution. */ - /* Transpose the matrix. */ - #include "transpose.h" - transpose_3xpow2(a, R, C); -#endif - - return 1; -} - -/* backward transform, sign = 1; transform length = 3 * 2**n */ -int -inv_four_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) -{ - mpd_size_t R = 3; /* number of rows */ - mpd_size_t C = n / 3; /* number of columns */ - mpd_uint_t w3table[3]; - mpd_uint_t kernel, w0, w1, wstep; - mpd_uint_t *s, *p0, *p1, *p2; - mpd_uint_t umod; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_size_t i, k; - - - assert(n >= 48); - assert(n <= 3*MPD_MAXTRANSFORM_2N); - - -#if 0 - /* An unordered transform is sufficient for convolution. */ - /* Transpose the matrix, producing an R*C matrix. */ - #include "transpose.h" - transpose_3xpow2(a, C, R); -#endif - - /* Length C transform on the rows. */ - for (s = a; s < a+n; s += C) { - if (!inv_six_step_fnt(s, C, modnum)) { - return 0; - } - } - - /* Multiply each matrix element (addressed by i*C+k) by r**(i*k). */ - SETMODULUS(modnum); - kernel = _mpd_getkernel(n, 1, modnum); - for (i = 1; i < R; i++) { - w0 = 1; - w1 = POWMOD(kernel, i); - wstep = MULMOD(w1, w1); - for (k = 0; k < C; k += 2) { - mpd_uint_t x0 = a[i*C+k]; - mpd_uint_t x1 = a[i*C+k+1]; - MULMOD2(&x0, w0, &x1, w1); - MULMOD2C(&w0, &w1, wstep); - a[i*C+k] = x0; - a[i*C+k+1] = x1; - } - } - - /* Length R transform on the columns. */ - _mpd_init_w3table(w3table, 1, modnum); - for (p0=a, p1=p0+C, p2=p0+2*C; p0<a+C; p0++,p1++,p2++) { - - SIZE3_NTT(p0, p1, p2, w3table); - } - - return 1; -} diff --git a/Modules/_decimal/libmpdec/fourstep.h b/Modules/_decimal/libmpdec/fourstep.h deleted file mode 100644 index 5ffb6fcc8ecd0d5..000000000000000 --- a/Modules/_decimal/libmpdec/fourstep.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_FOURSTEP_H_ -#define LIBMPDEC_FOURSTEP_H_ - - -#include "mpdecimal.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -int four_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum); -int inv_four_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_FOURSTEP_H_ */ diff --git a/Modules/_decimal/libmpdec/io.c b/Modules/_decimal/libmpdec/io.c deleted file mode 100644 index bdcca001659bc00..000000000000000 --- a/Modules/_decimal/libmpdec/io.c +++ /dev/null @@ -1,1606 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> -#include <ctype.h> -#include <errno.h> -#include <limits.h> -#include <locale.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "io.h" -#include "typearith.h" - - -/* This file contains functions for decimal <-> string conversions, including - PEP-3101 formatting for numeric types. */ - - -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 7 - #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" - #pragma GCC diagnostic ignored "-Wmisleading-indentation" - #pragma GCC diagnostic ignored "-Warray-bounds" -#endif - - -/* - * Work around the behavior of tolower() and strcasecmp() in certain - * locales. For example, in tr_TR.utf8: - * - * tolower((unsigned char)'I') == 'I' - * - * u is the exact uppercase version of l; n is strlen(l) or strlen(l)+1 - */ -static inline int -_mpd_strneq(const char *s, const char *l, const char *u, size_t n) -{ - while (--n != SIZE_MAX) { - if (*s != *l && *s != *u) { - return 0; - } - s++; u++; l++; - } - - return 1; -} - -static mpd_ssize_t -strtoexp(const char *s) -{ - char *end; - mpd_ssize_t retval; - - errno = 0; - retval = mpd_strtossize(s, &end, 10); - if (errno == 0 && !(*s != '\0' && *end == '\0')) - errno = EINVAL; - - return retval; -} - -/* - * Scan 'len' words. The most significant word contains 'r' digits, - * the remaining words are full words. Skip dpoint. The string 's' must - * consist of digits and an optional single decimal point at 'dpoint'. - */ -static void -string_to_coeff(mpd_uint_t *data, const char *s, const char *dpoint, int r, - size_t len) -{ - int j; - - if (r > 0) { - data[--len] = 0; - for (j = 0; j < r; j++, s++) { - if (s == dpoint) s++; - data[len] = 10 * data[len] + (*s - '0'); - } - } - - while (--len != SIZE_MAX) { - data[len] = 0; - for (j = 0; j < MPD_RDIGITS; j++, s++) { - if (s == dpoint) s++; - data[len] = 10 * data[len] + (*s - '0'); - } - } -} - -/* - * Partially verify a numeric string of the form: - * - * [cdigits][.][cdigits][eE][+-][edigits] - * - * If successful, return a pointer to the location of the first - * relevant coefficient digit. This digit is either non-zero or - * part of one of the following patterns: - * - * ["0\x00", "0.\x00", "0.E", "0.e", "0E", "0e"] - * - * The locations of a single optional dot or indicator are stored - * in 'dpoint' and 'exp'. - * - * The end of the string is stored in 'end'. If an indicator [eE] - * occurs without trailing [edigits], the condition is caught - * later by strtoexp(). - */ -static const char * -scan_dpoint_exp(const char *s, const char **dpoint, const char **exp, - const char **end) -{ - const char *coeff = NULL; - - *dpoint = NULL; - *exp = NULL; - for (; *s != '\0'; s++) { - switch (*s) { - case '.': - if (*dpoint != NULL || *exp != NULL) - return NULL; - *dpoint = s; - break; - case 'E': case 'e': - if (*exp != NULL) - return NULL; - *exp = s; - if (*(s+1) == '+' || *(s+1) == '-') - s++; - break; - default: - if (!isdigit((unsigned char)*s)) - return NULL; - if (coeff == NULL && *exp == NULL) { - if (*s == '0') { - if (!isdigit((unsigned char)*(s+1))) - if (!(*(s+1) == '.' && - isdigit((unsigned char)*(s+2)))) - coeff = s; - } - else { - coeff = s; - } - } - break; - - } - } - - *end = s; - return coeff; -} - -/* scan the payload of a NaN */ -static const char * -scan_payload(const char *s, const char **end) -{ - const char *coeff; - - while (*s == '0') - s++; - coeff = s; - - while (isdigit((unsigned char)*s)) - s++; - *end = s; - - return (*s == '\0') ? coeff : NULL; -} - -/* convert a character string to a decimal */ -void -mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_ssize_t q, r, len; - const char *coeff, *end; - const char *dpoint = NULL, *exp = NULL; - size_t digits; - uint8_t sign = MPD_POS; - - mpd_set_flags(dec, 0); - dec->len = 0; - dec->exp = 0; - - /* sign */ - if (*s == '+') { - s++; - } - else if (*s == '-') { - mpd_set_negative(dec); - sign = MPD_NEG; - s++; - } - - if (_mpd_strneq(s, "nan", "NAN", 3)) { /* NaN */ - s += 3; - mpd_setspecial(dec, sign, MPD_NAN); - if (*s == '\0') - return; - /* validate payload: digits only */ - if ((coeff = scan_payload(s, &end)) == NULL) - goto conversion_error; - /* payload consists entirely of zeros */ - if (*coeff == '\0') - return; - digits = end - coeff; - /* prec >= 1, clamp is 0 or 1 */ - if (digits > (size_t)(ctx->prec-ctx->clamp)) - goto conversion_error; - } /* sNaN */ - else if (_mpd_strneq(s, "snan", "SNAN", 4)) { - s += 4; - mpd_setspecial(dec, sign, MPD_SNAN); - if (*s == '\0') - return; - /* validate payload: digits only */ - if ((coeff = scan_payload(s, &end)) == NULL) - goto conversion_error; - /* payload consists entirely of zeros */ - if (*coeff == '\0') - return; - digits = end - coeff; - if (digits > (size_t)(ctx->prec-ctx->clamp)) - goto conversion_error; - } - else if (_mpd_strneq(s, "inf", "INF", 3)) { - s += 3; - if (*s == '\0' || _mpd_strneq(s, "inity", "INITY", 6)) { - /* numeric-value: infinity */ - mpd_setspecial(dec, sign, MPD_INF); - return; - } - goto conversion_error; - } - else { - /* scan for start of coefficient, decimal point, indicator, end */ - if ((coeff = scan_dpoint_exp(s, &dpoint, &exp, &end)) == NULL) - goto conversion_error; - - /* numeric-value: [exponent-part] */ - if (exp) { - /* exponent-part */ - end = exp; exp++; - dec->exp = strtoexp(exp); - if (errno) { - if (!(errno == ERANGE && - (dec->exp == MPD_SSIZE_MAX || - dec->exp == MPD_SSIZE_MIN))) - goto conversion_error; - } - } - - digits = end - coeff; - if (dpoint) { - size_t fracdigits = end-dpoint-1; - if (dpoint > coeff) digits--; - - if (fracdigits > MPD_MAX_PREC) { - goto conversion_error; - } - if (dec->exp < MPD_SSIZE_MIN+(mpd_ssize_t)fracdigits) { - dec->exp = MPD_SSIZE_MIN; - } - else { - dec->exp -= (mpd_ssize_t)fracdigits; - } - } - if (digits > MPD_MAX_PREC) { - goto conversion_error; - } - if (dec->exp > MPD_EXP_INF) { - dec->exp = MPD_EXP_INF; - } - if (dec->exp == MPD_SSIZE_MIN) { - dec->exp = MPD_SSIZE_MIN+1; - } - } - - _mpd_idiv_word(&q, &r, (mpd_ssize_t)digits, MPD_RDIGITS); - - len = (r == 0) ? q : q+1; - if (len == 0) { - goto conversion_error; /* GCOV_NOT_REACHED */ - } - if (!mpd_qresize(dec, len, status)) { - mpd_seterror(dec, MPD_Malloc_error, status); - return; - } - dec->len = len; - - string_to_coeff(dec->data, coeff, dpoint, (int)r, len); - - mpd_setdigits(dec); - mpd_qfinalize(dec, ctx, status); - return; - -conversion_error: - /* standard wants a positive NaN */ - mpd_seterror(dec, MPD_Conversion_syntax, status); -} - -/* convert a character string to a decimal, use a maxcontext for conversion */ -void -mpd_qset_string_exact(mpd_t *dec, const char *s, uint32_t *status) -{ - mpd_context_t maxcontext; - - mpd_maxcontext(&maxcontext); - mpd_qset_string(dec, s, &maxcontext, status); - - if (*status & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { - /* we want exact results */ - mpd_seterror(dec, MPD_Invalid_operation, status); - } - *status &= MPD_Errors; -} - -/* Print word x with n decimal digits to string s. dot is either NULL - or the location of a decimal point. */ -#define EXTRACT_DIGIT(s, x, d, dot) \ - if (s == dot) *s++ = '.'; *s++ = '0' + (char)(x / d); x %= d -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 12 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wstringop-overflow" -#endif -static inline char * -word_to_string(char *s, mpd_uint_t x, int n, char *dot) -{ - switch(n) { -#ifdef CONFIG_64 - case 20: EXTRACT_DIGIT(s, x, 10000000000000000000ULL, dot); /* GCOV_NOT_REACHED */ - case 19: EXTRACT_DIGIT(s, x, 1000000000000000000ULL, dot); - case 18: EXTRACT_DIGIT(s, x, 100000000000000000ULL, dot); - case 17: EXTRACT_DIGIT(s, x, 10000000000000000ULL, dot); - case 16: EXTRACT_DIGIT(s, x, 1000000000000000ULL, dot); - case 15: EXTRACT_DIGIT(s, x, 100000000000000ULL, dot); - case 14: EXTRACT_DIGIT(s, x, 10000000000000ULL, dot); - case 13: EXTRACT_DIGIT(s, x, 1000000000000ULL, dot); - case 12: EXTRACT_DIGIT(s, x, 100000000000ULL, dot); - case 11: EXTRACT_DIGIT(s, x, 10000000000ULL, dot); -#endif - case 10: EXTRACT_DIGIT(s, x, 1000000000UL, dot); - case 9: EXTRACT_DIGIT(s, x, 100000000UL, dot); - case 8: EXTRACT_DIGIT(s, x, 10000000UL, dot); - case 7: EXTRACT_DIGIT(s, x, 1000000UL, dot); - case 6: EXTRACT_DIGIT(s, x, 100000UL, dot); - case 5: EXTRACT_DIGIT(s, x, 10000UL, dot); - case 4: EXTRACT_DIGIT(s, x, 1000UL, dot); - case 3: EXTRACT_DIGIT(s, x, 100UL, dot); - case 2: EXTRACT_DIGIT(s, x, 10UL, dot); - default: if (s == dot) *s++ = '.'; *s++ = '0' + (char)x; - } - - *s = '\0'; - return s; -} -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 12 - #pragma GCC diagnostic pop -#endif - -/* Print exponent x to string s. Undefined for MPD_SSIZE_MIN. */ -static inline char * -exp_to_string(char *s, mpd_ssize_t x) -{ - char sign = '+'; - - if (x < 0) { - sign = '-'; - x = -x; - } - *s++ = sign; - - return word_to_string(s, x, mpd_word_digits(x), NULL); -} - -/* Print the coefficient of dec to string s. len(dec) > 0. */ -static inline char * -coeff_to_string(char *s, const mpd_t *dec) -{ - mpd_uint_t x; - mpd_ssize_t i; - - /* most significant word */ - x = mpd_msword(dec); - s = word_to_string(s, x, mpd_word_digits(x), NULL); - - /* remaining full words */ - for (i=dec->len-2; i >= 0; --i) { - x = dec->data[i]; - s = word_to_string(s, x, MPD_RDIGITS, NULL); - } - - return s; -} - -/* Print the coefficient of dec to string s. len(dec) > 0. dot is either - NULL or a pointer to the location of a decimal point. */ -static inline char * -coeff_to_string_dot(char *s, char *dot, const mpd_t *dec) -{ - mpd_uint_t x; - mpd_ssize_t i; - - /* most significant word */ - x = mpd_msword(dec); - s = word_to_string(s, x, mpd_word_digits(x), dot); - - /* remaining full words */ - for (i=dec->len-2; i >= 0; --i) { - x = dec->data[i]; - s = word_to_string(s, x, MPD_RDIGITS, dot); - } - - return s; -} - -/* Format type */ -#define MPD_FMT_LOWER 0x00000000 -#define MPD_FMT_UPPER 0x00000001 -#define MPD_FMT_TOSCI 0x00000002 -#define MPD_FMT_TOENG 0x00000004 -#define MPD_FMT_EXP 0x00000008 -#define MPD_FMT_FIXED 0x00000010 -#define MPD_FMT_PERCENT 0x00000020 -#define MPD_FMT_SIGN_SPACE 0x00000040 -#define MPD_FMT_SIGN_PLUS 0x00000080 - -/* Default place of the decimal point for MPD_FMT_TOSCI, MPD_FMT_EXP */ -#define MPD_DEFAULT_DOTPLACE 1 - -/* - * Set *result to the string representation of a decimal. Return the length - * of *result, not including the terminating '\0' character. - * - * Formatting is done according to 'flags'. A return value of -1 with *result - * set to NULL indicates MPD_Malloc_error. - * - * 'dplace' is the default place of the decimal point. It is always set to - * MPD_DEFAULT_DOTPLACE except for zeros in combination with MPD_FMT_EXP. - */ -static mpd_ssize_t -_mpd_to_string(char **result, const mpd_t *dec, int flags, mpd_ssize_t dplace) -{ - char *decstring = NULL, *cp = NULL; - mpd_ssize_t ldigits; - mpd_ssize_t mem = 0, k; - - if (mpd_isspecial(dec)) { - - mem = sizeof "-Infinity%"; - if (mpd_isnan(dec) && dec->len > 0) { - /* diagnostic code */ - mem += dec->digits; - } - cp = decstring = mpd_alloc(mem, sizeof *decstring); - if (cp == NULL) { - *result = NULL; - return -1; - } - - if (mpd_isnegative(dec)) { - *cp++ = '-'; - } - else if (flags&MPD_FMT_SIGN_SPACE) { - *cp++ = ' '; - } - else if (flags&MPD_FMT_SIGN_PLUS) { - *cp++ = '+'; - } - - if (mpd_isnan(dec)) { - if (mpd_isqnan(dec)) { - strcpy(cp, "NaN"); - cp += 3; - } - else { - strcpy(cp, "sNaN"); - cp += 4; - } - if (dec->len > 0) { /* diagnostic code */ - cp = coeff_to_string(cp, dec); - } - } - else if (mpd_isinfinite(dec)) { - strcpy(cp, "Infinity"); - cp += 8; - } - else { /* debug */ - abort(); /* GCOV_NOT_REACHED */ - } - } - else { - assert(dec->len > 0); - - /* - * For easier manipulation of the decimal point's location - * and the exponent that is finally printed, the number is - * rescaled to a virtual representation with exp = 0. Here - * ldigits denotes the number of decimal digits to the left - * of the decimal point and remains constant once initialized. - * - * dplace is the location of the decimal point relative to - * the start of the coefficient. Note that 3) always holds - * when dplace is shifted. - * - * 1) ldigits := dec->digits - dec->exp - * 2) dplace := ldigits (initially) - * 3) exp := ldigits - dplace (initially exp = 0) - * - * 0.00000_.____._____000000. - * ^ ^ ^ ^ - * | | | | - * | | | `- dplace >= digits - * | | `- dplace in the middle of the coefficient - * | ` dplace = 1 (after the first coefficient digit) - * `- dplace <= 0 - */ - - ldigits = dec->digits + dec->exp; - - if (flags&MPD_FMT_EXP) { - ; - } - else if (flags&MPD_FMT_FIXED || (dec->exp <= 0 && ldigits > -6)) { - /* MPD_FMT_FIXED: always use fixed point notation. - * MPD_FMT_TOSCI, MPD_FMT_TOENG: for a certain range, - * override exponent notation. */ - dplace = ldigits; - } - else if (flags&MPD_FMT_TOENG) { - if (mpd_iszero(dec)) { - /* If the exponent is divisible by three, - * dplace = 1. Otherwise, move dplace one - * or two places to the left. */ - dplace = -1 + mod_mpd_ssize_t(dec->exp+2, 3); - } - else { /* ldigits-1 is the adjusted exponent, which - * should be divisible by three. If not, move - * dplace one or two places to the right. */ - dplace += mod_mpd_ssize_t(ldigits-1, 3); - } - } - - /* - * Basic space requirements: - * - * [-][.][coeffdigits][E][-][expdigits+1][%]['\0'] - * - * If the decimal point lies outside of the coefficient digits, - * space is adjusted accordingly. - */ - if (dplace <= 0) { - mem = -dplace + dec->digits + 2; - } - else if (dplace >= dec->digits) { - mem = dplace; - } - else { - mem = dec->digits; - } - mem += (MPD_EXPDIGITS+1+6); - - cp = decstring = mpd_alloc(mem, sizeof *decstring); - if (cp == NULL) { - *result = NULL; - return -1; - } - - - if (mpd_isnegative(dec)) { - *cp++ = '-'; - } - else if (flags&MPD_FMT_SIGN_SPACE) { - *cp++ = ' '; - } - else if (flags&MPD_FMT_SIGN_PLUS) { - *cp++ = '+'; - } - - if (dplace <= 0) { - /* space: -dplace+dec->digits+2 */ - *cp++ = '0'; - *cp++ = '.'; - for (k = 0; k < -dplace; k++) { - *cp++ = '0'; - } - cp = coeff_to_string(cp, dec); - } - else if (dplace >= dec->digits) { - /* space: dplace */ - cp = coeff_to_string(cp, dec); - for (k = 0; k < dplace-dec->digits; k++) { - *cp++ = '0'; - } - } - else { - /* space: dec->digits+1 */ - cp = coeff_to_string_dot(cp, cp+dplace, dec); - } - - /* - * Conditions for printing an exponent: - * - * MPD_FMT_TOSCI, MPD_FMT_TOENG: only if ldigits != dplace - * MPD_FMT_FIXED: never (ldigits == dplace) - * MPD_FMT_EXP: always - */ - if (ldigits != dplace || flags&MPD_FMT_EXP) { - /* space: expdigits+2 */ - *cp++ = (flags&MPD_FMT_UPPER) ? 'E' : 'e'; - cp = exp_to_string(cp, ldigits-dplace); - } - } - - if (flags&MPD_FMT_PERCENT) { - *cp++ = '%'; - } - - assert(cp < decstring+mem); - assert(cp-decstring < MPD_SSIZE_MAX); - - *cp = '\0'; - *result = decstring; - return (mpd_ssize_t)(cp-decstring); -} - -char * -mpd_to_sci(const mpd_t *dec, int fmt) -{ - char *res; - int flags = MPD_FMT_TOSCI; - - flags |= fmt ? MPD_FMT_UPPER : MPD_FMT_LOWER; - (void)_mpd_to_string(&res, dec, flags, MPD_DEFAULT_DOTPLACE); - return res; -} - -char * -mpd_to_eng(const mpd_t *dec, int fmt) -{ - char *res; - int flags = MPD_FMT_TOENG; - - flags |= fmt ? MPD_FMT_UPPER : MPD_FMT_LOWER; - (void)_mpd_to_string(&res, dec, flags, MPD_DEFAULT_DOTPLACE); - return res; -} - -mpd_ssize_t -mpd_to_sci_size(char **res, const mpd_t *dec, int fmt) -{ - int flags = MPD_FMT_TOSCI; - - flags |= fmt ? MPD_FMT_UPPER : MPD_FMT_LOWER; - return _mpd_to_string(res, dec, flags, MPD_DEFAULT_DOTPLACE); -} - -mpd_ssize_t -mpd_to_eng_size(char **res, const mpd_t *dec, int fmt) -{ - int flags = MPD_FMT_TOENG; - - flags |= fmt ? MPD_FMT_UPPER : MPD_FMT_LOWER; - return _mpd_to_string(res, dec, flags, MPD_DEFAULT_DOTPLACE); -} - -/* Copy a single UTF-8 char to dest. See: The Unicode Standard, version 5.2, - chapter 3.9: Well-formed UTF-8 byte sequences. */ -static int -_mpd_copy_utf8(char dest[5], const char *s) -{ - const unsigned char *cp = (const unsigned char *)s; - unsigned char lb, ub; - int count, i; - - - if (*cp == 0) { - /* empty string */ - dest[0] = '\0'; - return 0; - } - else if (*cp <= 0x7f) { - /* ascii */ - dest[0] = *cp; - dest[1] = '\0'; - return 1; - } - else if (0xc2 <= *cp && *cp <= 0xdf) { - lb = 0x80; ub = 0xbf; - count = 2; - } - else if (*cp == 0xe0) { - lb = 0xa0; ub = 0xbf; - count = 3; - } - else if (*cp <= 0xec) { - lb = 0x80; ub = 0xbf; - count = 3; - } - else if (*cp == 0xed) { - lb = 0x80; ub = 0x9f; - count = 3; - } - else if (*cp <= 0xef) { - lb = 0x80; ub = 0xbf; - count = 3; - } - else if (*cp == 0xf0) { - lb = 0x90; ub = 0xbf; - count = 4; - } - else if (*cp <= 0xf3) { - lb = 0x80; ub = 0xbf; - count = 4; - } - else if (*cp == 0xf4) { - lb = 0x80; ub = 0x8f; - count = 4; - } - else { - /* invalid */ - goto error; - } - - dest[0] = *cp++; - if (*cp < lb || ub < *cp) { - goto error; - } - dest[1] = *cp++; - for (i = 2; i < count; i++) { - if (*cp < 0x80 || 0xbf < *cp) { - goto error; - } - dest[i] = *cp++; - } - dest[i] = '\0'; - - return count; - -error: - dest[0] = '\0'; - return -1; -} - -int -mpd_validate_lconv(mpd_spec_t *spec) -{ - size_t n; -#if CHAR_MAX == SCHAR_MAX - const char *cp = spec->grouping; - while (*cp != '\0') { - if (*cp++ < 0) { - return -1; - } - } -#endif - n = strlen(spec->dot); - if (n == 0 || n > 4) { - return -1; - } - if (strlen(spec->sep) > 4) { - return -1; - } - - return 0; -} - -int -mpd_parse_fmt_str(mpd_spec_t *spec, const char *fmt, int caps) -{ - char *cp = (char *)fmt; - int have_align = 0, n; - - /* defaults */ - spec->min_width = 0; - spec->prec = -1; - spec->type = caps ? 'G' : 'g'; - spec->align = '>'; - spec->sign = '-'; - spec->dot = ""; - spec->sep = ""; - spec->grouping = ""; - - - /* presume that the first character is a UTF-8 fill character */ - if ((n = _mpd_copy_utf8(spec->fill, cp)) < 0) { - return 0; - } - - /* alignment directive, prefixed by a fill character */ - if (*cp && (*(cp+n) == '<' || *(cp+n) == '>' || - *(cp+n) == '=' || *(cp+n) == '^')) { - cp += n; - spec->align = *cp++; - have_align = 1; - } /* alignment directive */ - else { - /* default fill character */ - spec->fill[0] = ' '; - spec->fill[1] = '\0'; - if (*cp == '<' || *cp == '>' || - *cp == '=' || *cp == '^') { - spec->align = *cp++; - have_align = 1; - } - } - - /* sign formatting */ - if (*cp == '+' || *cp == '-' || *cp == ' ') { - spec->sign = *cp++; - } - - /* zero padding */ - if (*cp == '0') { - /* zero padding implies alignment, which should not be - * specified twice. */ - if (have_align) { - return 0; - } - spec->align = 'z'; - spec->fill[0] = *cp++; - spec->fill[1] = '\0'; - } - - /* minimum width */ - if (isdigit((unsigned char)*cp)) { - if (*cp == '0') { - return 0; - } - errno = 0; - spec->min_width = mpd_strtossize(cp, &cp, 10); - if (errno == ERANGE || errno == EINVAL) { - return 0; - } - } - - /* thousands separator */ - if (*cp == ',') { - spec->dot = "."; - spec->sep = ","; - spec->grouping = "\003\003"; - cp++; - } - - /* fraction digits or significant digits */ - if (*cp == '.') { - cp++; - if (!isdigit((unsigned char)*cp)) { - return 0; - } - errno = 0; - spec->prec = mpd_strtossize(cp, &cp, 10); - if (errno == ERANGE || errno == EINVAL) { - return 0; - } - } - - /* type */ - if (*cp == 'E' || *cp == 'e' || *cp == 'F' || *cp == 'f' || - *cp == 'G' || *cp == 'g' || *cp == '%') { - spec->type = *cp++; - } - else if (*cp == 'N' || *cp == 'n') { - /* locale specific conversion */ - struct lconv *lc; - /* separator has already been specified */ - if (*spec->sep) { - return 0; - } - spec->type = *cp++; - spec->type = (spec->type == 'N') ? 'G' : 'g'; - lc = localeconv(); - spec->dot = lc->decimal_point; - spec->sep = lc->thousands_sep; - spec->grouping = lc->grouping; - if (mpd_validate_lconv(spec) < 0) { - return 0; /* GCOV_NOT_REACHED */ - } - } - - /* check correctness */ - if (*cp != '\0') { - return 0; - } - - return 1; -} - -/* - * The following functions assume that spec->min_width <= MPD_MAX_PREC, which - * is made sure in mpd_qformat_spec. Then, even with a spec that inserts a - * four-byte separator after each digit, nbytes in the following struct - * cannot overflow. - */ - -/* Multibyte string */ -typedef struct { - mpd_ssize_t nbytes; /* length in bytes */ - mpd_ssize_t nchars; /* length in chars */ - mpd_ssize_t cur; /* current write index */ - char *data; -} mpd_mbstr_t; - -static inline void -_mpd_bcopy(char *dest, const char *src, mpd_ssize_t n) -{ - while (--n >= 0) { - dest[n] = src[n]; - } -} - -static inline void -_mbstr_copy_char(mpd_mbstr_t *dest, const char *src, mpd_ssize_t n) -{ - dest->nbytes += n; - dest->nchars += (n > 0 ? 1 : 0); - dest->cur -= n; - - if (dest->data != NULL) { - _mpd_bcopy(dest->data+dest->cur, src, n); - } -} - -static inline void -_mbstr_copy_ascii(mpd_mbstr_t *dest, const char *src, mpd_ssize_t n) -{ - dest->nbytes += n; - dest->nchars += n; - dest->cur -= n; - - if (dest->data != NULL) { - _mpd_bcopy(dest->data+dest->cur, src, n); - } -} - -static inline void -_mbstr_copy_pad(mpd_mbstr_t *dest, mpd_ssize_t n) -{ - dest->nbytes += n; - dest->nchars += n; - dest->cur -= n; - - if (dest->data != NULL) { - char *cp = dest->data + dest->cur; - while (--n >= 0) { - cp[n] = '0'; - } - } -} - -/* - * Copy a numeric string to dest->data, adding separators in the integer - * part according to spec->grouping. If leading zero padding is enabled - * and the result is smaller than spec->min_width, continue adding zeros - * and separators until the minimum width is reached. - * - * The final length of dest->data is stored in dest->nbytes. The number - * of UTF-8 characters is stored in dest->nchars. - * - * First run (dest->data == NULL): determine the length of the result - * string and store it in dest->nbytes. - * - * Second run (write to dest->data): data is written in chunks and in - * reverse order, starting with the rest of the numeric string. - */ -static void -_mpd_add_sep_dot(mpd_mbstr_t *dest, - const char *sign, /* location of optional sign */ - const char *src, mpd_ssize_t n_src, /* integer part and length */ - const char *dot, /* location of optional decimal point */ - const char *rest, mpd_ssize_t n_rest, /* remaining part and length */ - const mpd_spec_t *spec) -{ - mpd_ssize_t n_sep, n_sign, consume; - const char *g; - int pad = 0; - - n_sign = sign ? 1 : 0; - n_sep = (mpd_ssize_t)strlen(spec->sep); - /* Initial write index: set to location of '\0' in the output string. - * Irrelevant for the first run. */ - dest->cur = dest->nbytes; - dest->nbytes = dest->nchars = 0; - - _mbstr_copy_ascii(dest, rest, n_rest); - - if (dot) { - _mbstr_copy_char(dest, dot, (mpd_ssize_t)strlen(dot)); - } - - g = spec->grouping; - consume = *g; - while (1) { - /* If the group length is 0 or CHAR_MAX or greater than the - * number of source bytes, consume all remaining bytes. */ - if (*g == 0 || *g == CHAR_MAX || consume > n_src) { - consume = n_src; - } - n_src -= consume; - if (pad) { - _mbstr_copy_pad(dest, consume); - } - else { - _mbstr_copy_ascii(dest, src+n_src, consume); - } - - if (n_src == 0) { - /* Either the real source of intpart digits or the virtual - * source of padding zeros is exhausted. */ - if (spec->align == 'z' && - dest->nchars + n_sign < spec->min_width) { - /* Zero padding is set and length < min_width: - * Generate n_src additional characters. */ - n_src = spec->min_width - (dest->nchars + n_sign); - /* Next iteration: - * case *g == 0 || *g == CHAR_MAX: - * consume all padding characters - * case consume < g*: - * fill remainder of current group - * case consume == g* - * copying is a no-op */ - consume = *g - consume; - /* Switch on virtual source of zeros. */ - pad = 1; - continue; - } - break; - } - - if (n_sep > 0) { - /* If padding is switched on, separators are counted - * as padding characters. This rule does not apply if - * the separator would be the first character of the - * result string. */ - if (pad && n_src > 1) n_src -= 1; - _mbstr_copy_char(dest, spec->sep, n_sep); - } - - /* If non-NUL, use the next value for grouping. */ - if (*g && *(g+1)) g++; - consume = *g; - } - - if (sign) { - _mbstr_copy_ascii(dest, sign, 1); - } - - if (dest->data) { - dest->data[dest->nbytes] = '\0'; - } -} - -/* - * Convert a numeric-string to its locale-specific appearance. - * The string must have one of these forms: - * - * 1) [sign] digits [exponent-part] - * 2) [sign] digits '.' [digits] [exponent-part] - * - * Not allowed, since _mpd_to_string() never returns this form: - * - * 3) [sign] '.' digits [exponent-part] - * - * Input: result->data := original numeric string (ASCII) - * result->bytes := strlen(result->data) - * result->nchars := strlen(result->data) - * - * Output: result->data := modified or original string - * result->bytes := strlen(result->data) - * result->nchars := number of characters (possibly UTF-8) - */ -static int -_mpd_apply_lconv(mpd_mbstr_t *result, const mpd_spec_t *spec, uint32_t *status) -{ - const char *sign = NULL, *intpart = NULL, *dot = NULL; - const char *rest, *dp; - char *decstring; - mpd_ssize_t n_int, n_rest; - - /* original numeric string */ - dp = result->data; - - /* sign */ - if (*dp == '+' || *dp == '-' || *dp == ' ') { - sign = dp++; - } - /* integer part */ - assert(isdigit((unsigned char)*dp)); - intpart = dp++; - while (isdigit((unsigned char)*dp)) { - dp++; - } - n_int = (mpd_ssize_t)(dp-intpart); - /* decimal point */ - if (*dp == '.') { - dp++; dot = spec->dot; - } - /* rest */ - rest = dp; - n_rest = result->nbytes - (mpd_ssize_t)(dp-result->data); - - if (dot == NULL && (*spec->sep == '\0' || *spec->grouping == '\0')) { - /* _mpd_add_sep_dot() would not change anything */ - return 1; - } - - /* Determine the size of the new decimal string after inserting the - * decimal point, optional separators and optional padding. */ - decstring = result->data; - result->data = NULL; - _mpd_add_sep_dot(result, sign, intpart, n_int, dot, - rest, n_rest, spec); - - result->data = mpd_alloc(result->nbytes+1, 1); - if (result->data == NULL) { - *status |= MPD_Malloc_error; - mpd_free(decstring); - return 0; - } - - /* Perform actual writes. */ - _mpd_add_sep_dot(result, sign, intpart, n_int, dot, - rest, n_rest, spec); - - mpd_free(decstring); - return 1; -} - -/* Add padding to the formatted string if necessary. */ -static int -_mpd_add_pad(mpd_mbstr_t *result, const mpd_spec_t *spec, uint32_t *status) -{ - if (result->nchars < spec->min_width) { - mpd_ssize_t add_chars, add_bytes; - size_t lpad = 0, rpad = 0; - size_t n_fill, len, i, j; - char align = spec->align; - uint8_t err = 0; - char *cp; - - n_fill = strlen(spec->fill); - add_chars = (spec->min_width - result->nchars); - /* max value: MPD_MAX_PREC * 4 */ - add_bytes = add_chars * (mpd_ssize_t)n_fill; - - cp = result->data = mpd_realloc(result->data, - result->nbytes+add_bytes+1, - sizeof *result->data, &err); - if (err) { - *status |= MPD_Malloc_error; - mpd_free(result->data); - return 0; - } - - if (align == 'z') { - align = '='; - } - - if (align == '<') { - rpad = add_chars; - } - else if (align == '>' || align == '=') { - lpad = add_chars; - } - else { /* align == '^' */ - lpad = add_chars/2; - rpad = add_chars-lpad; - } - - len = result->nbytes; - if (align == '=' && (*cp == '-' || *cp == '+' || *cp == ' ')) { - /* leave sign in the leading position */ - cp++; len--; - } - - memmove(cp+n_fill*lpad, cp, len); - for (i = 0; i < lpad; i++) { - for (j = 0; j < n_fill; j++) { - cp[i*n_fill+j] = spec->fill[j]; - } - } - cp += (n_fill*lpad + len); - for (i = 0; i < rpad; i++) { - for (j = 0; j < n_fill; j++) { - cp[i*n_fill+j] = spec->fill[j]; - } - } - - result->nbytes += add_bytes; - result->nchars += add_chars; - result->data[result->nbytes] = '\0'; - } - - return 1; -} - -/* Round a number to prec digits. The adjusted exponent stays the same - or increases by one if rounding up crosses a power of ten boundary. - If result->digits would exceed MPD_MAX_PREC+1, MPD_Invalid_operation - is set and the result is NaN. */ -static inline void -_mpd_round(mpd_t *result, const mpd_t *a, mpd_ssize_t prec, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t exp = a->exp + a->digits - prec; - - if (prec <= 0) { - mpd_seterror(result, MPD_Invalid_operation, status); /* GCOV_NOT_REACHED */ - return; /* GCOV_NOT_REACHED */ - } - if (mpd_isspecial(a) || mpd_iszero(a)) { - mpd_qcopy(result, a, status); /* GCOV_NOT_REACHED */ - return; /* GCOV_NOT_REACHED */ - } - - mpd_qrescale_fmt(result, a, exp, ctx, status); - if (result->digits > prec) { - mpd_qrescale_fmt(result, result, exp+1, ctx, status); - } -} - -/* - * Return the string representation of an mpd_t, formatted according to 'spec'. - * The format specification is assumed to be valid. Memory errors are indicated - * as usual. This function is quiet. - */ -char * -mpd_qformat_spec(const mpd_t *dec, const mpd_spec_t *spec, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_uint_t dt[MPD_MINALLOC_MAX]; - mpd_t tmp = {MPD_STATIC|MPD_STATIC_DATA,0,0,0,MPD_MINALLOC_MAX,dt}; - mpd_ssize_t dplace = MPD_DEFAULT_DOTPLACE; - mpd_mbstr_t result; - mpd_spec_t stackspec; - char type = spec->type; - int flags = 0; - - - if (spec->min_width > MPD_MAX_PREC) { - *status |= MPD_Invalid_operation; - return NULL; - } - - if (isupper((unsigned char)type)) { - type = (char)tolower((unsigned char)type); - flags |= MPD_FMT_UPPER; - } - if (spec->sign == ' ') { - flags |= MPD_FMT_SIGN_SPACE; - } - else if (spec->sign == '+') { - flags |= MPD_FMT_SIGN_PLUS; - } - - if (mpd_isspecial(dec)) { - if (spec->align == 'z') { - stackspec = *spec; - stackspec.fill[0] = ' '; - stackspec.fill[1] = '\0'; - stackspec.align = '>'; - spec = &stackspec; - } - assert(strlen(spec->fill) == 1); /* annotation for scan-build */ - if (type == '%') { - flags |= MPD_FMT_PERCENT; - } - } - else { - uint32_t workstatus = 0; - mpd_ssize_t prec; - - switch (type) { - case 'g': flags |= MPD_FMT_TOSCI; break; - case 'e': flags |= MPD_FMT_EXP; break; - case '%': flags |= MPD_FMT_PERCENT; - if (!mpd_qcopy(&tmp, dec, status)) { - return NULL; - } - tmp.exp += 2; - dec = &tmp; - type = 'f'; /* fall through */ - case 'f': flags |= MPD_FMT_FIXED; break; - default: abort(); /* debug: GCOV_NOT_REACHED */ - } - - if (spec->prec >= 0) { - if (spec->prec > MPD_MAX_PREC) { - *status |= MPD_Invalid_operation; - goto error; - } - - switch (type) { - case 'g': - prec = (spec->prec == 0) ? 1 : spec->prec; - if (dec->digits > prec) { - _mpd_round(&tmp, dec, prec, ctx, - &workstatus); - dec = &tmp; - } - break; - case 'e': - if (mpd_iszero(dec)) { - dplace = 1-spec->prec; - } - else { - _mpd_round(&tmp, dec, spec->prec+1, ctx, - &workstatus); - dec = &tmp; - } - break; - case 'f': - mpd_qrescale(&tmp, dec, -spec->prec, ctx, - &workstatus); - dec = &tmp; - break; - } - } - - if (type == 'f') { - if (mpd_iszero(dec) && dec->exp > 0) { - mpd_qrescale(&tmp, dec, 0, ctx, &workstatus); - dec = &tmp; - } - } - - if (workstatus&MPD_Errors) { - *status |= (workstatus&MPD_Errors); - goto error; - } - } - - /* - * At this point, for all scaled or non-scaled decimals: - * 1) 1 <= digits <= MAX_PREC+1 - * 2) adjexp(scaled) = adjexp(orig) [+1] - * 3) case 'g': MIN_ETINY <= exp <= MAX_EMAX+1 - * case 'e': MIN_ETINY-MAX_PREC <= exp <= MAX_EMAX+1 - * case 'f': MIN_ETINY <= exp <= MAX_EMAX+1 - * 4) max memory alloc in _mpd_to_string: - * case 'g': MAX_PREC+36 - * case 'e': MAX_PREC+36 - * case 'f': 2*MPD_MAX_PREC+30 - */ - result.nbytes = _mpd_to_string(&result.data, dec, flags, dplace); - result.nchars = result.nbytes; - if (result.nbytes < 0) { - *status |= MPD_Malloc_error; - goto error; - } - - if (*spec->dot != '\0' && !mpd_isspecial(dec)) { - if (result.nchars > MPD_MAX_PREC+36) { - /* Since a group length of one is not explicitly - * disallowed, ensure that it is always possible to - * insert a four byte separator after each digit. */ - *status |= MPD_Invalid_operation; - mpd_free(result.data); - goto error; - } - if (!_mpd_apply_lconv(&result, spec, status)) { - goto error; - } - } - - if (spec->min_width) { - if (!_mpd_add_pad(&result, spec, status)) { - goto error; - } - } - - mpd_del(&tmp); - return result.data; - -error: - mpd_del(&tmp); - return NULL; -} - -char * -mpd_qformat(const mpd_t *dec, const char *fmt, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_spec_t spec; - - if (!mpd_parse_fmt_str(&spec, fmt, 1)) { - *status |= MPD_Invalid_operation; - return NULL; - } - - return mpd_qformat_spec(dec, &spec, ctx, status); -} - -/* - * The specification has a *condition* called Invalid_operation and an - * IEEE *signal* called Invalid_operation. The former corresponds to - * MPD_Invalid_operation, the latter to MPD_IEEE_Invalid_operation. - * MPD_IEEE_Invalid_operation comprises the following conditions: - * - * [MPD_Conversion_syntax, MPD_Division_impossible, MPD_Division_undefined, - * MPD_Fpu_error, MPD_Invalid_context, MPD_Invalid_operation, - * MPD_Malloc_error] - * - * In the following functions, 'flag' denotes the condition, 'signal' - * denotes the IEEE signal. - */ - -static const char *mpd_flag_string[MPD_NUM_FLAGS] = { - "Clamped", - "Conversion_syntax", - "Division_by_zero", - "Division_impossible", - "Division_undefined", - "Fpu_error", - "Inexact", - "Invalid_context", - "Invalid_operation", - "Malloc_error", - "Not_implemented", - "Overflow", - "Rounded", - "Subnormal", - "Underflow", -}; - -static const char *mpd_signal_string[MPD_NUM_FLAGS] = { - "Clamped", - "IEEE_Invalid_operation", - "Division_by_zero", - "IEEE_Invalid_operation", - "IEEE_Invalid_operation", - "IEEE_Invalid_operation", - "Inexact", - "IEEE_Invalid_operation", - "IEEE_Invalid_operation", - "IEEE_Invalid_operation", - "Not_implemented", - "Overflow", - "Rounded", - "Subnormal", - "Underflow", -}; - -/* print conditions to buffer, separated by spaces */ -int -mpd_snprint_flags(char *dest, int nmemb, uint32_t flags) -{ - char *cp; - int n, j; - - assert(nmemb >= MPD_MAX_FLAG_STRING); - - *dest = '\0'; cp = dest; - for (j = 0; j < MPD_NUM_FLAGS; j++) { - if (flags & (1U<<j)) { - n = snprintf(cp, nmemb, "%s ", mpd_flag_string[j]); - if (n < 0 || n >= nmemb) return -1; - cp += n; nmemb -= n; - } - } - - if (cp != dest) { - *(--cp) = '\0'; - } - - return (int)(cp-dest); -} - -/* print conditions to buffer, in list form */ -int -mpd_lsnprint_flags(char *dest, int nmemb, uint32_t flags, const char *flag_string[]) -{ - char *cp; - int n, j; - - assert(nmemb >= MPD_MAX_FLAG_LIST); - if (flag_string == NULL) { - flag_string = mpd_flag_string; - } - - *dest = '['; - *(dest+1) = '\0'; - cp = dest+1; - --nmemb; - - for (j = 0; j < MPD_NUM_FLAGS; j++) { - if (flags & (1U<<j)) { - n = snprintf(cp, nmemb, "%s, ", flag_string[j]); - if (n < 0 || n >= nmemb) return -1; - cp += n; nmemb -= n; - } - } - - /* erase the last ", " */ - if (cp != dest+1) { - cp -= 2; - } - - *cp++ = ']'; - *cp = '\0'; - - return (int)(cp-dest); /* strlen, without NUL terminator */ -} - -/* print signals to buffer, in list form */ -int -mpd_lsnprint_signals(char *dest, int nmemb, uint32_t flags, const char *signal_string[]) -{ - char *cp; - int n, j; - int ieee_invalid_done = 0; - - assert(nmemb >= MPD_MAX_SIGNAL_LIST); - if (signal_string == NULL) { - signal_string = mpd_signal_string; - } - - *dest = '['; - *(dest+1) = '\0'; - cp = dest+1; - --nmemb; - - for (j = 0; j < MPD_NUM_FLAGS; j++) { - uint32_t f = flags & (1U<<j); - if (f) { - if (f&MPD_IEEE_Invalid_operation) { - if (ieee_invalid_done) { - continue; - } - ieee_invalid_done = 1; - } - n = snprintf(cp, nmemb, "%s, ", signal_string[j]); - if (n < 0 || n >= nmemb) return -1; - cp += n; nmemb -= n; - } - } - - /* erase the last ", " */ - if (cp != dest+1) { - cp -= 2; - } - - *cp++ = ']'; - *cp = '\0'; - - return (int)(cp-dest); /* strlen, without NUL terminator */ -} - -/* The following two functions are mainly intended for debugging. */ -void -mpd_fprint(FILE *file, const mpd_t *dec) -{ - char *decstring; - - decstring = mpd_to_sci(dec, 1); - if (decstring != NULL) { - fprintf(file, "%s\n", decstring); - mpd_free(decstring); - } - else { - fputs("mpd_fprint: output error\n", file); /* GCOV_NOT_REACHED */ - } -} - -void -mpd_print(const mpd_t *dec) -{ - char *decstring; - - decstring = mpd_to_sci(dec, 1); - if (decstring != NULL) { - printf("%s\n", decstring); - mpd_free(decstring); - } - else { - fputs("mpd_fprint: output error\n", stderr); /* GCOV_NOT_REACHED */ - } -} diff --git a/Modules/_decimal/libmpdec/io.h b/Modules/_decimal/libmpdec/io.h deleted file mode 100644 index 79d7c05ce369c66..000000000000000 --- a/Modules/_decimal/libmpdec/io.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_IO_H_ -#define LIBMPDEC_IO_H_ - - -#include "mpdecimal.h" - -#include <stdint.h> - - -#if SIZE_MAX == MPD_SIZE_MAX - #define mpd_strtossize _mpd_strtossize -#else -#include <errno.h> - -static inline mpd_ssize_t -mpd_strtossize(const char *s, char **end, int base) -{ - int64_t retval; - - errno = 0; - retval = _mpd_strtossize(s, end, base); - if (errno == 0 && (retval > MPD_SSIZE_MAX || retval < MPD_SSIZE_MIN)) { - errno = ERANGE; - } - if (errno == ERANGE) { - return (retval < 0) ? MPD_SSIZE_MIN : MPD_SSIZE_MAX; - } - - return (mpd_ssize_t)retval; -} -#endif - - -#endif /* LIBMPDEC_IO_H_ */ diff --git a/Modules/_decimal/libmpdec/literature/REFERENCES.txt b/Modules/_decimal/libmpdec/literature/REFERENCES.txt deleted file mode 100644 index 9ed5782656a31ba..000000000000000 --- a/Modules/_decimal/libmpdec/literature/REFERENCES.txt +++ /dev/null @@ -1,51 +0,0 @@ - - -This document contains links to the literature used in the process of -creating the library. The list is probably not complete. - - -Mike Cowlishaw: General Decimal Arithmetic Specification -http://speleotrove.com/decimal/decarith.html - - -Jean-Michel Muller: On the definition of ulp (x) -lara.inist.fr/bitstream/2332/518/1/LIP-RR2005-09.pdf - - -T. E. Hull, A. Abrham: Properly rounded variable precision square root -http://portal.acm.org/citation.cfm?id=214413 - - -T. E. Hull, A. Abrham: Variable precision exponential function -http://portal.acm.org/citation.cfm?id=6498 - - -Roman E. Maeder: Storage allocation for the Karatsuba integer multiplication -algorithm. http://www.springerlink.com/content/w15058mj6v59t565/ - - -J. M. Pollard: The fast Fourier transform in a finite field -http://www.ams.org/journals/mcom/1971-25-114/S0025-5718-1971-0301966-0/home.html - - -David H. Bailey: FFTs in External or Hierarchical Memory -http://crd.lbl.gov/~dhbailey/dhbpapers/ - - -W. Morven Gentleman: Matrix Multiplication and Fast Fourier Transforms -http://www.alcatel-lucent.com/bstj/vol47-1968/articles/bstj47-6-1099.pdf - - -Mikko Tommila: Apfloat documentation -http://www.apfloat.org/apfloat/2.41/apfloat.pdf - - -Joerg Arndt: "Matters Computational" -http://www.jjj.de/fxt/ - - -Karl Hasselstrom: Fast Division of Large Integers -www.treskal.com/kalle/exjobb/original-report.pdf - - - diff --git a/Modules/_decimal/libmpdec/literature/bignum.txt b/Modules/_decimal/libmpdec/literature/bignum.txt deleted file mode 100644 index f34ff67c612ab4e..000000000000000 --- a/Modules/_decimal/libmpdec/literature/bignum.txt +++ /dev/null @@ -1,83 +0,0 @@ - - -Bignum support (Fast Number Theoretic Transform or FNT): -======================================================== - -Bignum arithmetic in libmpdec uses the scheme for fast convolution -of integer sequences from: - -J. M. Pollard: The fast Fourier transform in a finite field -http://www.ams.org/journals/mcom/1971-25-114/S0025-5718-1971-0301966-0/home.html - - -The transform in a finite field can be used for convolution in the same -way as the Fourier Transform. The main advantages of the Number Theoretic -Transform are that it is both exact and very memory efficient. - - -Convolution in pseudo-code: -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - fnt_convolute(a, b): - x = fnt(a) # forward transform of a - y = fnt(b) # forward transform of b - z = pairwise multiply x[i] and y[i] - result = inv_fnt(z) # backward transform of z. - - -Extending the maximum transform length (Chinese Remainder Theorem): -------------------------------------------------------------------- - -The maximum transform length is quite limited when using a single -prime field. However, it is possible to use multiple primes and -recover the result using the Chinese Remainder Theorem. - - -Multiplication in pseudo-code: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - _mpd_fntmul(u, v): - c1 = fnt_convolute(u, v, P1) # convolute modulo prime1 - c2 = fnt_convolute(u, v, P2) # convolute modulo prime2 - c3 = fnt_convolute(u, v, P3) # convolute modulo prime3 - result = crt3(c1, c2, c3) # Chinese Remainder Theorem - - -Optimized transform functions: ------------------------------- - -There are three different fnt() functions: - - std_fnt: "standard" decimation in frequency transform for array lengths - of 2**n. Performs well up to 1024 words. - - sixstep: Cache-friendly algorithm for array lengths of 2**n. Outperforms - std_fnt for large arrays. - - fourstep: Algorithm for array lengths of 3 * 2**n. Also cache friendly - in large parts. - - -List of bignum-only files: --------------------------- - -Functions from these files are only used in _mpd_fntmul(). - - umodarith.h -> fast low level routines for unsigned modular arithmetic - numbertheory.c -> routines for setting up the FNT - difradix2.c -> decimation in frequency transform, used as the - "base case" by the following three files: - - fnt.c -> standard transform for smaller arrays - sixstep.c -> transform large arrays of length 2**n - fourstep.c -> transform arrays of length 3 * 2**n - - convolute.c -> do the actual fast convolution, using one of - the three transform functions. - transpose.c -> transpositions needed for the sixstep algorithm. - crt.c -> Chinese Remainder Theorem: use information from three - transforms modulo three different primes to get the - final result. - - - diff --git a/Modules/_decimal/libmpdec/literature/fnt.py b/Modules/_decimal/libmpdec/literature/fnt.py deleted file mode 100644 index c1285a565db96bd..000000000000000 --- a/Modules/_decimal/libmpdec/literature/fnt.py +++ /dev/null @@ -1,208 +0,0 @@ -# -# Copyright (c) 2008-2020 Stefan Krah. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -# SUCH DAMAGE. -# - - -###################################################################### -# This file lists and checks some of the constants and limits used # -# in libmpdec's Number Theoretic Transform. At the end of the file # -# there is an example function for the plain DFT transform. # -###################################################################### - - -# -# Number theoretic transforms are done in subfields of F(p). P[i] -# are the primes, D[i] = P[i] - 1 are highly composite and w[i] -# are the respective primitive roots of F(p). -# -# The strategy is to convolute two coefficients modulo all three -# primes, then use the Chinese Remainder Theorem on the three -# result arrays to recover the result in the usual base RADIX -# form. -# - -# ====================================================================== -# Primitive roots -# ====================================================================== - -# -# Verify primitive roots: -# -# For a prime field, r is a primitive root if and only if for all prime -# factors f of p-1, r**((p-1)/f) =/= 1 (mod p). -# -def prod(F, E): - """Check that the factorization of P-1 is correct. F is the list of - factors of P-1, E lists the number of occurrences of each factor.""" - x = 1 - for y, z in zip(F, E): - x *= y**z - return x - -def is_primitive_root(r, p, factors, exponents): - """Check if r is a primitive root of F(p).""" - if p != prod(factors, exponents) + 1: - return False - for f in factors: - q, control = divmod(p-1, f) - if control != 0: - return False - if pow(r, q, p) == 1: - return False - return True - - -# ================================================================= -# Constants and limits for the 64-bit version -# ================================================================= - -RADIX = 10**19 - -# Primes P1, P2 and P3: -P = [2**64-2**32+1, 2**64-2**34+1, 2**64-2**40+1] - -# P-1, highly composite. The transform length d is variable and -# must divide D = P-1. Since all D are divisible by 3 * 2**32, -# transform lengths can be 2**n or 3 * 2**n (where n <= 32). -D = [2**32 * 3 * (5 * 17 * 257 * 65537), - 2**34 * 3**2 * (7 * 11 * 31 * 151 * 331), - 2**40 * 3**2 * (5 * 7 * 13 * 17 * 241)] - -# Prime factors of P-1 and their exponents: -F = [(2,3,5,17,257,65537), (2,3,7,11,31,151,331), (2,3,5,7,13,17,241)] -E = [(32,1,1,1,1,1), (34,2,1,1,1,1,1), (40,2,1,1,1,1,1)] - -# Maximum transform length for 2**n. Above that only 3 * 2**31 -# or 3 * 2**32 are possible. -MPD_MAXTRANSFORM_2N = 2**32 - - -# Limits in the terminology of Pollard's paper: -m2 = (MPD_MAXTRANSFORM_2N * 3) // 2 # Maximum length of the smaller array. -M1 = M2 = RADIX-1 # Maximum value per single word. -L = m2 * M1 * M2 -P[0] * P[1] * P[2] > 2 * L - - -# Primitive roots of F(P1), F(P2) and F(P3): -w = [7, 10, 19] - -# The primitive roots are correct: -for i in range(3): - if not is_primitive_root(w[i], P[i], F[i], E[i]): - print("FAIL") - - -# ================================================================= -# Constants and limits for the 32-bit version -# ================================================================= - -RADIX = 10**9 - -# Primes P1, P2 and P3: -P = [2113929217, 2013265921, 1811939329] - -# P-1, highly composite. All D = P-1 are divisible by 3 * 2**25, -# allowing for transform lengths up to 3 * 2**25 words. -D = [2**25 * 3**2 * 7, - 2**27 * 3 * 5, - 2**26 * 3**3] - -# Prime factors of P-1 and their exponents: -F = [(2,3,7), (2,3,5), (2,3)] -E = [(25,2,1), (27,1,1), (26,3)] - -# Maximum transform length for 2**n. Above that only 3 * 2**24 or -# 3 * 2**25 are possible. -MPD_MAXTRANSFORM_2N = 2**25 - - -# Limits in the terminology of Pollard's paper: -m2 = (MPD_MAXTRANSFORM_2N * 3) // 2 # Maximum length of the smaller array. -M1 = M2 = RADIX-1 # Maximum value per single word. -L = m2 * M1 * M2 -P[0] * P[1] * P[2] > 2 * L - - -# Primitive roots of F(P1), F(P2) and F(P3): -w = [5, 31, 13] - -# The primitive roots are correct: -for i in range(3): - if not is_primitive_root(w[i], P[i], F[i], E[i]): - print("FAIL") - - -# ====================================================================== -# Example transform using a single prime -# ====================================================================== - -def ntt(lst, dir): - """Perform a transform on the elements of lst. len(lst) must - be 2**n or 3 * 2**n, where n <= 25. This is the slow DFT.""" - p = 2113929217 # prime - d = len(lst) # transform length - d_prime = pow(d, (p-2), p) # inverse of d - xi = (p-1)//d - w = 5 # primitive root of F(p) - r = pow(w, xi, p) # primitive root of the subfield - r_prime = pow(w, (p-1-xi), p) # inverse of r - if dir == 1: # forward transform - a = lst # input array - A = [0] * d # transformed values - for i in range(d): - s = 0 - for j in range(d): - s += a[j] * pow(r, i*j, p) - A[i] = s % p - return A - elif dir == -1: # backward transform - A = lst # input array - a = [0] * d # transformed values - for j in range(d): - s = 0 - for i in range(d): - s += A[i] * pow(r_prime, i*j, p) - a[j] = (d_prime * s) % p - return a - -def ntt_convolute(a, b): - """convolute arrays a and b.""" - assert(len(a) == len(b)) - x = ntt(a, 1) - y = ntt(b, 1) - for i in range(len(a)): - y[i] = y[i] * x[i] - r = ntt(y, -1) - return r - - -# Example: Two arrays representing 21 and 81 in little-endian: -a = [1, 2, 0, 0] -b = [1, 8, 0, 0] - -assert(ntt_convolute(a, b) == [1, 10, 16, 0]) -assert(21 * 81 == (1*10**0 + 10*10**1 + 16*10**2 + 0*10**3)) diff --git a/Modules/_decimal/libmpdec/literature/matrix-transform.txt b/Modules/_decimal/libmpdec/literature/matrix-transform.txt deleted file mode 100644 index 6e7ad7420909fd2..000000000000000 --- a/Modules/_decimal/libmpdec/literature/matrix-transform.txt +++ /dev/null @@ -1,256 +0,0 @@ - - -(* Copyright (c) 2011-2020 Stefan Krah. All rights reserved. *) - - -The Matrix Fourier Transform: -============================= - -In libmpdec, the Matrix Fourier Transform [1] is called four-step transform -after a variant that appears in [2]. The algorithm requires that the input -array can be viewed as an R*C matrix. - -All operations are done modulo p. For readability, the proofs drop all -instances of (mod p). - - -Algorithm four-step (forward transform): ----------------------------------------- - - a := input array - d := len(a) = R * C - p := prime - w := primitive root of unity of the prime field - r := w**((p-1)/d) - A := output array - - 1) Apply a length R FNT to each column. - - 2) Multiply each matrix element (addressed by j*C+m) by r**(j*m). - - 3) Apply a length C FNT to each row. - - 4) Transpose the matrix. - - -Proof (forward transform): --------------------------- - - The algorithm can be derived starting from the regular definition of - the finite-field transform of length d: - - d-1 - ,---- - \ - A[k] = | a[l] * r**(k * l) - / - `---- - l = 0 - - - The sum can be rearranged into the sum of the sums of columns: - - C-1 R-1 - ,---- ,---- - \ \ - = | | a[i * C + j] * r**(k * (i * C + j)) - / / - `---- `---- - j = 0 i = 0 - - - Extracting a constant from the inner sum: - - C-1 R-1 - ,---- ,---- - \ \ - = | r**k*j * | a[i * C + j] * r**(k * i * C) - / / - `---- `---- - j = 0 i = 0 - - - Without any loss of generality, let k = n * R + m, - where n < C and m < R: - - C-1 R-1 - ,---- ,---- - \ \ - A[n*R+m] = | r**(R*n*j) * r**(m*j) * | a[i*C+j] * r**(R*C*n*i) * r**(C*m*i) - / / - `---- `---- - j = 0 i = 0 - - - Since r = w ** ((p-1) / (R*C)): - - a) r**(R*C*n*i) = w**((p-1)*n*i) = 1 - - b) r**(C*m*i) = w**((p-1) / R) ** (m*i) = r_R ** (m*i) - - c) r**(R*n*j) = w**((p-1) / C) ** (n*j) = r_C ** (n*j) - - r_R := root of the subfield of length R. - r_C := root of the subfield of length C. - - - C-1 R-1 - ,---- ,---- - \ \ - A[n*R+m] = | r_C**(n*j) * [ r**(m*j) * | a[i*C+j] * r_R**(m*i) ] - / ^ / - `---- | `---- 1) transform the columns - j = 0 | i = 0 - ^ | - | `-- 2) multiply - | - `-- 3) transform the rows - - - Note that the entire RHS is a function of n and m and that the results - for each pair (n, m) are stored in Fortran order. - - Let the term in square brackets be f(m, j). Step 1) and 2) precalculate - the term for all (m, j). After that, the original matrix is now a lookup - table with the mth element in the jth column at location m * C + j. - - Let the complete RHS be g(m, n). Step 3) does an in-place transform of - length n on all rows. After that, the original matrix is now a lookup - table with the mth element in the nth column at location m * C + n. - - But each (m, n) pair should be written to location n * R + m. Therefore, - step 4) transposes the result of step 3). - - - -Algorithm four-step (inverse transform): ----------------------------------------- - - A := input array - d := len(A) = R * C - p := prime - d' := d**(p-2) # inverse of d - w := primitive root of unity of the prime field - r := w**((p-1)/d) # root of the subfield - r' := w**((p-1) - (p-1)/d) # inverse of r - a := output array - - 0) View the matrix as a C*R matrix. - - 1) Transpose the matrix, producing an R*C matrix. - - 2) Apply a length C FNT to each row. - - 3) Multiply each matrix element (addressed by i*C+n) by r**(i*n). - - 4) Apply a length R FNT to each column. - - -Proof (inverse transform): --------------------------- - - The algorithm can be derived starting from the regular definition of - the finite-field inverse transform of length d: - - d-1 - ,---- - \ - a[k] = d' * | A[l] * r' ** (k * l) - / - `---- - l = 0 - - - The sum can be rearranged into the sum of the sums of columns. Note - that at this stage we still have a C*R matrix, so C denotes the number - of rows: - - R-1 C-1 - ,---- ,---- - \ \ - = d' * | | a[j * R + i] * r' ** (k * (j * R + i)) - / / - `---- `---- - i = 0 j = 0 - - - Extracting a constant from the inner sum: - - R-1 C-1 - ,---- ,---- - \ \ - = d' * | r' ** (k*i) * | a[j * R + i] * r' ** (k * j * R) - / / - `---- `---- - i = 0 j = 0 - - - Without any loss of generality, let k = m * C + n, - where m < R and n < C: - - R-1 C-1 - ,---- ,---- - \ \ - A[m*C+n] = d' * | r' ** (C*m*i) * r' ** (n*i) * | a[j*R+i] * r' ** (R*C*m*j) * r' ** (R*n*j) - / / - `---- `---- - i = 0 j = 0 - - - Since r' = w**((p-1) - (p-1)/d) and d = R*C: - - a) r' ** (R*C*m*j) = w**((p-1)*R*C*m*j - (p-1)*m*j) = 1 - - b) r' ** (C*m*i) = w**((p-1)*C - (p-1)/R) ** (m*i) = r_R' ** (m*i) - - c) r' ** (R*n*j) = r_C' ** (n*j) - - d) d' = d**(p-2) = (R*C) ** (p-2) = R**(p-2) * C**(p-2) = R' * C' - - r_R' := inverse of the root of the subfield of length R. - r_C' := inverse of the root of the subfield of length C. - R' := inverse of R - C' := inverse of C - - - R-1 C-1 - ,---- ,---- 2) transform the rows of a^T - \ \ - A[m*C+n] = R' * | r_R' ** (m*i) * [ r' ** (n*i) * C' * | a[j*R+i] * r_C' ** (n*j) ] - / ^ / ^ - `---- | `---- | - i = 0 | j = 0 | - ^ | `-- 1) Transpose input matrix - | `-- 3) multiply to address elements by - | i * C + j - `-- 3) transform the columns - - - - Note that the entire RHS is a function of m and n and that the results - for each pair (m, n) are stored in C order. - - Let the term in square brackets be f(n, i). Without step 1), the sum - would perform a length C transform on the columns of the input matrix. - This is a) inefficient and b) the results are needed in C order, so - step 1) exchanges rows and columns. - - Step 2) and 3) precalculate f(n, i) for all (n, i). After that, the - original matrix is now a lookup table with the ith element in the nth - column at location i * C + n. - - Let the complete RHS be g(m, n). Step 4) does an in-place transform of - length m on all columns. After that, the original matrix is now a lookup - table with the mth element in the nth column at location m * C + n, - which means that all A[k] = A[m * C + n] are in the correct order. - - --- - - [1] Joerg Arndt: "Matters Computational" - http://www.jjj.de/fxt/ - [2] David H. Bailey: FFTs in External or Hierarchical Memory - http://crd.lbl.gov/~dhbailey/dhbpapers/ - - - diff --git a/Modules/_decimal/libmpdec/literature/mulmod-64.txt b/Modules/_decimal/libmpdec/literature/mulmod-64.txt deleted file mode 100644 index fa967bf95e3030b..000000000000000 --- a/Modules/_decimal/libmpdec/literature/mulmod-64.txt +++ /dev/null @@ -1,127 +0,0 @@ - - -(* Copyright (c) 2011-2020 Stefan Krah. All rights reserved. *) - - -========================================================================== - Calculate (a * b) % p using special primes -========================================================================== - -A description of the algorithm can be found in the apfloat manual by -Tommila [1]. - - -Definitions: ------------- - -In the whole document, "==" stands for "is congruent with". - -Result of a * b in terms of high/low words: - - (1) hi * 2**64 + lo = a * b - -Special primes: - - (2) p = 2**64 - z + 1, where z = 2**n - -Single step modular reduction: - - (3) R(hi, lo) = hi * z - hi + lo - - -Strategy: ---------- - - a) Set (hi, lo) to the result of a * b. - - b) Set (hi', lo') to the result of R(hi, lo). - - c) Repeat step b) until 0 <= hi' * 2**64 + lo' < 2*p. - - d) If the result is less than p, return lo'. Otherwise return lo' - p. - - -The reduction step b) preserves congruence: -------------------------------------------- - - hi * 2**64 + lo == hi * z - hi + lo (mod p) - - Proof: - ~~~~~~ - - hi * 2**64 + lo = (2**64 - z + 1) * hi + z * hi - hi + lo - - = p * hi + z * hi - hi + lo - - == z * hi - hi + lo (mod p) - - -Maximum numbers of step b): ---------------------------- - -# To avoid unnecessary formalism, define: - -def R(hi, lo, z): - return divmod(hi * z - hi + lo, 2**64) - -# For simplicity, assume hi=2**64-1, lo=2**64-1 after the -# initial multiplication a * b. This is of course impossible -# but certainly covers all cases. - -# Then, for p1: -hi=2**64-1; lo=2**64-1; z=2**32 -p1 = 2**64 - z + 1 - -hi, lo = R(hi, lo, z) # First reduction -hi, lo = R(hi, lo, z) # Second reduction -hi * 2**64 + lo < 2 * p1 # True - -# For p2: -hi=2**64-1; lo=2**64-1; z=2**34 -p2 = 2**64 - z + 1 - -hi, lo = R(hi, lo, z) # First reduction -hi, lo = R(hi, lo, z) # Second reduction -hi, lo = R(hi, lo, z) # Third reduction -hi * 2**64 + lo < 2 * p2 # True - -# For p3: -hi=2**64-1; lo=2**64-1; z=2**40 -p3 = 2**64 - z + 1 - -hi, lo = R(hi, lo, z) # First reduction -hi, lo = R(hi, lo, z) # Second reduction -hi, lo = R(hi, lo, z) # Third reduction -hi * 2**64 + lo < 2 * p3 # True - - -Step d) preserves congruence and yields a result < p: ------------------------------------------------------ - - Case hi = 0: - - Case lo < p: trivial. - - Case lo >= p: - - lo == lo - p (mod p) # result is congruent - - p <= lo < 2*p -> 0 <= lo - p < p # result is in the correct range - - Case hi = 1: - - p < 2**64 /\ 2**64 + lo < 2*p -> lo < p # lo is always less than p - - 2**64 + lo == 2**64 + (lo - p) (mod p) # result is congruent - - = lo - p # exactly the same value as the previous RHS - # in uint64_t arithmetic. - - p < 2**64 + lo < 2*p -> 0 < 2**64 + (lo - p) < p # correct range - - - -[1] http://www.apfloat.org/apfloat/2.40/apfloat.pdf - - - diff --git a/Modules/_decimal/libmpdec/literature/mulmod-ppro.txt b/Modules/_decimal/libmpdec/literature/mulmod-ppro.txt deleted file mode 100644 index ba804e4b4e78641..000000000000000 --- a/Modules/_decimal/libmpdec/literature/mulmod-ppro.txt +++ /dev/null @@ -1,269 +0,0 @@ - - -(* Copyright (c) 2011-2020 Stefan Krah. All rights reserved. *) - - -======================================================================== - Calculate (a * b) % p using the 80-bit x87 FPU -======================================================================== - -A description of the algorithm can be found in the apfloat manual by -Tommila [1]. - -The proof follows an argument made by Granlund/Montgomery in [2]. - - -Definitions and assumptions: ----------------------------- - -The 80-bit extended precision format uses 64 bits for the significand: - - (1) F = 64 - -The modulus is prime and less than 2**31: - - (2) 2 <= p < 2**31 - -The factors are less than p: - - (3) 0 <= a < p - (4) 0 <= b < p - -The product a * b is less than 2**62 and is thus exact in 64 bits: - - (5) n = a * b - -The product can be represented in terms of quotient and remainder: - - (6) n = q * p + r - -Using (3), (4) and the fact that p is prime, the remainder is always -greater than zero: - - (7) 0 <= q < p /\ 1 <= r < p - - -Strategy: ---------- - -Precalculate the 80-bit long double inverse of p, with a maximum -relative error of 2**(1-F): - - (8) pinv = (long double)1.0 / p - -Calculate an estimate for q = floor(n/p). The multiplication has another -maximum relative error of 2**(1-F): - - (9) qest = n * pinv - -If we can show that q < qest < q+1, then trunc(qest) = q. It is then -easy to recover the remainder r. The complete algorithm is: - - a) Set the control word to 64-bit precision and truncation mode. - - b) n = a * b # Calculate exact product. - - c) qest = n * pinv # Calculate estimate for the quotient. - - d) q = (qest+2**63)-2**63 # Truncate qest to the exact quotient. - - f) r = n - q * p # Calculate remainder. - - -Proof for q < qest < q+1: -------------------------- - -Using the cumulative error, the error bounds for qest are: - - n n * (1 + 2**(1-F))**2 - (9) --------------------- <= qest <= --------------------- - p * (1 + 2**(1-F))**2 p - - - Lemma 1: - -------- - n q * p + r - (10) q < --------------------- = --------------------- - p * (1 + 2**(1-F))**2 p * (1 + 2**(1-F))**2 - - - Proof: - ~~~~~~ - - (I) q * p * (1 + 2**(1-F))**2 < q * p + r - - (II) q * p * 2**(2-F) + q * p * 2**(2-2*F) < r - - Using (1) and (7), it is sufficient to show that: - - (III) q * p * 2**(-62) + q * p * 2**(-126) < 1 <= r - - (III) can easily be verified by substituting the largest possible - values p = 2**31-1 and q = 2**31-2. - - The critical cases occur when r = 1, n = m * p + 1. These cases - can be exhaustively verified with a test program. - - - Lemma 2: - -------- - - n * (1 + 2**(1-F))**2 (q * p + r) * (1 + 2**(1-F))**2 - (11) --------------------- = ------------------------------- < q + 1 - p p - - Proof: - ~~~~~~ - - (I) (q * p + r) + (q * p + r) * 2**(2-F) + (q * p + r) * 2**(2-2*F) < q * p + p - - (II) (q * p + r) * 2**(2-F) + (q * p + r) * 2**(2-2*F) < p - r - - Using (1) and (7), it is sufficient to show that: - - (III) (q * p + r) * 2**(-62) + (q * p + r) * 2**(-126) < 1 <= p - r - - (III) can easily be verified by substituting the largest possible - values p = 2**31-1, q = 2**31-2 and r = 2**31-2. - - The critical cases occur when r = (p - 1), n = m * p - 1. These cases - can be exhaustively verified with a test program. - - -[1] http://www.apfloat.org/apfloat/2.40/apfloat.pdf -[2] http://gmplib.org/~tege/divcnst-pldi94.pdf - [Section 7: "Use of floating point"] - - - -(* Coq proof for (10) and (11) *) - -Require Import ZArith. -Require Import QArith. -Require Import Qpower. -Require Import Qabs. -Require Import Psatz. - -Open Scope Q_scope. - - -Ltac qreduce T := - rewrite <- (Qred_correct (T)); simpl (Qred (T)). - -Theorem Qlt_move_right : - forall x y z:Q, x + z < y <-> x < y - z. -Proof. - intros. - split. - intros. - psatzl Q. - intros. - psatzl Q. -Qed. - -Theorem Qlt_mult_by_z : - forall x y z:Q, 0 < z -> (x < y <-> x * z < y * z). -Proof. - intros. - split. - intros. - apply Qmult_lt_compat_r. trivial. trivial. - intros. - rewrite <- (Qdiv_mult_l x z). rewrite <- (Qdiv_mult_l y z). - apply Qmult_lt_compat_r. - apply Qlt_shift_inv_l. - trivial. psatzl Q. trivial. psatzl Q. psatzl Q. -Qed. - -Theorem Qle_mult_quad : - forall (a b c d:Q), - 0 <= a -> a <= c -> - 0 <= b -> b <= d -> - a * b <= c * d. - intros. - psatz Q. -Qed. - - -Theorem q_lt_qest: - forall (p q r:Q), - (0 < p) -> (p <= (2#1)^31 - 1) -> - (0 <= q) -> (q <= p - 1) -> - (1 <= r) -> (r <= p - 1) -> - q < (q * p + r) / (p * (1 + (2#1)^(-63))^2). -Proof. - intros. - rewrite Qlt_mult_by_z with (z := (p * (1 + (2#1)^(-63))^2)). - - unfold Qdiv. - rewrite <- Qmult_assoc. - rewrite (Qmult_comm (/ (p * (1 + (2 # 1) ^ (-63)) ^ 2)) (p * (1 + (2 # 1) ^ (-63)) ^ 2)). - rewrite Qmult_inv_r. - rewrite Qmult_1_r. - - assert (q * (p * (1 + (2 # 1) ^ (-63)) ^ 2) == q * p + (q * p) * ((2 # 1) ^ (-62) + (2 # 1) ^ (-126))). - qreduce ((1 + (2 # 1) ^ (-63)) ^ 2). - qreduce ((2 # 1) ^ (-62) + (2 # 1) ^ (-126)). - ring_simplify. - reflexivity. - rewrite H5. - - rewrite Qplus_comm. - rewrite Qlt_move_right. - ring_simplify (q * p + r - q * p). - qreduce ((2 # 1) ^ (-62) + (2 # 1) ^ (-126)). - - apply Qlt_le_trans with (y := 1). - rewrite Qlt_mult_by_z with (z := 85070591730234615865843651857942052864 # 18446744073709551617). - ring_simplify. - - apply Qle_lt_trans with (y := ((2 # 1) ^ 31 - (2#1)) * ((2 # 1) ^ 31 - 1)). - apply Qle_mult_quad. - assumption. psatzl Q. psatzl Q. psatzl Q. psatzl Q. psatzl Q. assumption. psatzl Q. psatzl Q. -Qed. - -Theorem qest_lt_qplus1: - forall (p q r:Q), - (0 < p) -> (p <= (2#1)^31 - 1) -> - (0 <= q) -> (q <= p - 1) -> - (1 <= r) -> (r <= p - 1) -> - ((q * p + r) * (1 + (2#1)^(-63))^2) / p < q + 1. -Proof. - intros. - rewrite Qlt_mult_by_z with (z := p). - - unfold Qdiv. - rewrite <- Qmult_assoc. - rewrite (Qmult_comm (/ p) p). - rewrite Qmult_inv_r. - rewrite Qmult_1_r. - - assert ((q * p + r) * (1 + (2 # 1) ^ (-63)) ^ 2 == q * p + r + (q * p + r) * ((2 # 1) ^ (-62) + (2 # 1) ^ (-126))). - qreduce ((1 + (2 # 1) ^ (-63)) ^ 2). - qreduce ((2 # 1) ^ (-62) + (2 # 1) ^ (-126)). - ring_simplify. reflexivity. - rewrite H5. - - rewrite <- Qplus_assoc. rewrite <- Qplus_comm. rewrite Qlt_move_right. - ring_simplify ((q + 1) * p - q * p). - - rewrite <- Qplus_comm. rewrite Qlt_move_right. - - apply Qlt_le_trans with (y := 1). - qreduce ((2 # 1) ^ (-62) + (2 # 1) ^ (-126)). - - rewrite Qlt_mult_by_z with (z := 85070591730234615865843651857942052864 # 18446744073709551617). - ring_simplify. - - ring_simplify in H0. - apply Qle_lt_trans with (y := (2147483646 # 1) * (2147483647 # 1) + (2147483646 # 1)). - - apply Qplus_le_compat. - apply Qle_mult_quad. - assumption. psatzl Q. auto with qarith. assumption. psatzl Q. - auto with qarith. auto with qarith. - psatzl Q. psatzl Q. assumption. -Qed. - - - diff --git a/Modules/_decimal/libmpdec/literature/six-step.txt b/Modules/_decimal/libmpdec/literature/six-step.txt deleted file mode 100644 index 852d5b0df8bf37b..000000000000000 --- a/Modules/_decimal/libmpdec/literature/six-step.txt +++ /dev/null @@ -1,63 +0,0 @@ - - -(* Copyright (c) 2011-2020 Stefan Krah. All rights reserved. *) - - -The Six Step Transform: -======================= - -In libmpdec, the six-step transform is the Matrix Fourier Transform (See -matrix-transform.txt) in disguise. It is called six-step transform after -a variant that appears in [1]. The algorithm requires that the input -array can be viewed as an R*C matrix. - - -Algorithm six-step (forward transform): ---------------------------------------- - - 1a) Transpose the matrix. - - 1b) Apply a length R FNT to each row. - - 1c) Transpose the matrix. - - 2) Multiply each matrix element (addressed by j*C+m) by r**(j*m). - - 3) Apply a length C FNT to each row. - - 4) Transpose the matrix. - -Note that steps 1a) - 1c) are exactly equivalent to step 1) of the Matrix -Fourier Transform. For large R, it is faster to transpose twice and do -a transform on the rows than to perform a column transpose directly. - - - -Algorithm six-step (inverse transform): ---------------------------------------- - - 0) View the matrix as a C*R matrix. - - 1) Transpose the matrix, producing an R*C matrix. - - 2) Apply a length C FNT to each row. - - 3) Multiply each matrix element (addressed by i*C+n) by r**(i*n). - - 4a) Transpose the matrix. - - 4b) Apply a length R FNT to each row. - - 4c) Transpose the matrix. - -Again, steps 4a) - 4c) are equivalent to step 4) of the Matrix Fourier -Transform. - - - --- - - [1] David H. Bailey: FFTs in External or Hierarchical Memory - http://crd.lbl.gov/~dhbailey/dhbpapers/ - - diff --git a/Modules/_decimal/libmpdec/literature/umodarith.lisp b/Modules/_decimal/libmpdec/literature/umodarith.lisp deleted file mode 100644 index d71f074a26dccaa..000000000000000 --- a/Modules/_decimal/libmpdec/literature/umodarith.lisp +++ /dev/null @@ -1,692 +0,0 @@ -; -; Copyright (c) 2008-2020 Stefan Krah. All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions -; are met: -; -; 1. Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; -; 2. Redistributions in binary form must reproduce the above copyright -; notice, this list of conditions and the following disclaimer in the -; documentation and/or other materials provided with the distribution. -; -; THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND -; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -; ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -; OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -; HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -; SUCH DAMAGE. -; - - -(in-package "ACL2") - -(include-book "arithmetic/top-with-meta" :dir :system) -(include-book "arithmetic-2/floor-mod/floor-mod" :dir :system) - - -;; ===================================================================== -;; Proofs for several functions in umodarith.h -;; ===================================================================== - - - -;; ===================================================================== -;; Helper theorems -;; ===================================================================== - -(defthm elim-mod-m<x<2*m - (implies (and (<= m x) - (< x (* 2 m)) - (rationalp x) (rationalp m)) - (equal (mod x m) - (+ x (- m))))) - -(defthm modaux-1a - (implies (and (< x m) (< 0 x) (< 0 m) - (rationalp x) (rationalp m)) - (equal (mod (- x) m) - (+ (- x) m)))) - -(defthm modaux-1b - (implies (and (< (- x) m) (< x 0) (< 0 m) - (rationalp x) (rationalp m)) - (equal (mod x m) - (+ x m))) - :hints (("Goal" :use ((:instance modaux-1a - (x (- x))))))) - -(defthm modaux-1c - (implies (and (< x m) (< 0 x) (< 0 m) - (rationalp x) (rationalp m)) - (equal (mod x m) - x))) - -(defthm modaux-2a - (implies (and (< 0 b) (< b m) - (natp x) (natp b) (natp m) - (< (mod (+ b x) m) b)) - (equal (mod (+ (- m) b x) m) - (+ (- m) b (mod x m))))) - -(defthm modaux-2b - (implies (and (< 0 b) (< b m) - (natp x) (natp b) (natp m) - (< (mod (+ b x) m) b)) - (equal (mod (+ b x) m) - (+ (- m) b (mod x m)))) - :hints (("Goal" :use (modaux-2a)))) - -(defthm linear-mod-1 - (implies (and (< x m) (< b m) - (natp x) (natp b) - (rationalp m)) - (equal (< x (mod (+ (- b) x) m)) - (< x b))) - :hints (("Goal" :use ((:instance modaux-1a - (x (+ b (- x)))))))) - -(defthm linear-mod-2 - (implies (and (< 0 b) (< b m) - (natp x) (natp b) - (natp m)) - (equal (< (mod x m) - (mod (+ (- b) x) m)) - (< (mod x m) b)))) - -(defthm linear-mod-3 - (implies (and (< x m) (< b m) - (natp x) (natp b) - (rationalp m)) - (equal (<= b (mod (+ b x) m)) - (< (+ b x) m))) - :hints (("Goal" :use ((:instance elim-mod-m<x<2*m - (x (+ b x))))))) - -(defthm modaux-2c - (implies (and (< 0 b) (< b m) - (natp x) (natp b) (natp m) - (<= b (mod (+ b x) m))) - (equal (mod (+ b x) m) - (+ b (mod x m)))) - :hints (("Subgoal *1/8''" :use (linear-mod-3)))) - -(defthmd modaux-2d - (implies (and (< x m) (< 0 x) (< 0 m) - (< (- m) b) (< b 0) (rationalp m) - (<= x (mod (+ b x) m)) - (rationalp x) (rationalp b)) - (equal (+ (- m) (mod (+ b x) m)) - (+ b x))) - :hints (("Goal" :cases ((<= 0 (+ b x)))) - ("Subgoal 2'" :use ((:instance modaux-1b - (x (+ b x))))))) - -(defthm mod-m-b - (implies (and (< 0 x) (< 0 b) (< 0 m) - (< x b) (< b m) - (natp x) (natp b) (natp m)) - (equal (mod (+ (mod (- x) m) b) m) - (mod (- x) b)))) - - -;; ===================================================================== -;; addmod, submod -;; ===================================================================== - -(defun addmod (a b m base) - (let* ((s (mod (+ a b) base)) - (s (if (< s a) (mod (- s m) base) s)) - (s (if (>= s m) (mod (- s m) base) s))) - s)) - -(defthmd addmod-correct - (implies (and (< 0 m) (< m base) - (< a m) (<= b m) - (natp m) (natp base) - (natp a) (natp b)) - (equal (addmod a b m base) - (mod (+ a b) m))) - :hints (("Goal" :cases ((<= base (+ a b)))) - ("Subgoal 2.1'" :use ((:instance elim-mod-m<x<2*m - (x (+ a b))))))) - -(defun submod (a b m base) - (let* ((d (mod (- a b) base)) - (d (if (< a d) (mod (+ d m) base) d))) - d)) - -(defthmd submod-aux1 - (implies (and (< a (mod (+ a (- b)) base)) - (< 0 base) (< a base) (<= b base) - (natp base) (natp a) (natp b)) - (< a b)) - :rule-classes :forward-chaining) - -(defthmd submod-aux2 - (implies (and (<= (mod (+ a (- b)) base) a) - (< 0 base) (< a base) (< b base) - (natp base) (natp a) (natp b)) - (<= b a)) - :rule-classes :forward-chaining) - -(defthmd submod-correct - (implies (and (< 0 m) (< m base) - (< a m) (<= b m) - (natp m) (natp base) - (natp a) (natp b)) - (equal (submod a b m base) - (mod (- a b) m))) - :hints (("Goal" :cases ((<= base (+ a b)))) - ("Subgoal 2.2" :use ((:instance submod-aux1))) - ("Subgoal 2.2'''" :cases ((and (< 0 (+ a (- b) m)) - (< (+ a (- b) m) m)))) - ("Subgoal 2.1" :use ((:instance submod-aux2))) - ("Subgoal 1.2" :use ((:instance submod-aux1))) - ("Subgoal 1.1" :use ((:instance submod-aux2))))) - - -(defun submod-2 (a b m base) - (let* ((d (mod (- a b) base)) - (d (if (< a b) (mod (+ d m) base) d))) - d)) - -(defthm submod-2-correct - (implies (and (< 0 m) (< m base) - (< a m) (<= b m) - (natp m) (natp base) - (natp a) (natp b)) - (equal (submod-2 a b m base) - (mod (- a b) m))) - :hints (("Subgoal 2'" :cases ((and (< 0 (+ a (- b) m)) - (< (+ a (- b) m) m)))))) - - -;; ========================================================================= -;; ext-submod is correct -;; ========================================================================= - -; a < 2*m, b < 2*m -(defun ext-submod (a b m base) - (let* ((a (if (>= a m) (- a m) a)) - (b (if (>= b m) (- b m) b)) - (d (mod (- a b) base)) - (d (if (< a b) (mod (+ d m) base) d))) - d)) - -; a < 2*m, b < 2*m -(defun ext-submod-2 (a b m base) - (let* ((a (mod a m)) - (b (mod b m)) - (d (mod (- a b) base)) - (d (if (< a b) (mod (+ d m) base) d))) - d)) - -(defthmd ext-submod-ext-submod-2-equal - (implies (and (< 0 m) (< m base) - (< a (* 2 m)) (< b (* 2 m)) - (natp m) (natp base) - (natp a) (natp b)) - (equal (ext-submod a b m base) - (ext-submod-2 a b m base)))) - -(defthmd ext-submod-2-correct - (implies (and (< 0 m) (< m base) - (< a (* 2 m)) (< b (* 2 m)) - (natp m) (natp base) - (natp a) (natp b)) - (equal (ext-submod-2 a b m base) - (mod (- a b) m)))) - - -;; ========================================================================= -;; dw-reduce is correct -;; ========================================================================= - -(defun dw-reduce (hi lo m base) - (let* ((r1 (mod hi m)) - (r2 (mod (+ (* r1 base) lo) m))) - r2)) - -(defthmd dw-reduce-correct - (implies (and (< 0 m) (< m base) - (< hi base) (< lo base) - (natp m) (natp base) - (natp hi) (natp lo)) - (equal (dw-reduce hi lo m base) - (mod (+ (* hi base) lo) m)))) - -(defthmd <=-multiply-both-sides-by-z - (implies (and (rationalp x) (rationalp y) - (< 0 z) (rationalp z)) - (equal (<= x y) - (<= (* z x) (* z y))))) - -(defthmd dw-reduce-aux1 - (implies (and (< 0 m) (< m base) - (natp m) (natp base) - (< lo base) (natp lo) - (< x m) (natp x)) - (< (+ lo (* base x)) (* base m))) - :hints (("Goal" :cases ((<= (+ x 1) m))) - ("Subgoal 1''" :cases ((<= (* base (+ x 1)) (* base m)))) - ("subgoal 1.2" :use ((:instance <=-multiply-both-sides-by-z - (x (+ 1 x)) - (y m) - (z base)))))) - -(defthm dw-reduce-aux2 - (implies (and (< x (* base m)) - (< 0 m) (< m base) - (natp m) (natp base) (natp x)) - (< (floor x m) base))) - -;; This is the necessary condition for using _mpd_div_words(). -(defthmd dw-reduce-second-quotient-fits-in-single-word - (implies (and (< 0 m) (< m base) - (< hi base) (< lo base) - (natp m) (natp base) - (natp hi) (natp lo) - (equal r1 (mod hi m))) - (< (floor (+ (* r1 base) lo) m) - base)) - :hints (("Goal" :cases ((< r1 m))) - ("Subgoal 1''" :cases ((< (+ lo (* base (mod hi m))) (* base m)))) - ("Subgoal 1.2" :use ((:instance dw-reduce-aux1 - (x (mod hi m))))))) - - -;; ========================================================================= -;; dw-submod is correct -;; ========================================================================= - -(defun dw-submod (a hi lo m base) - (let* ((r (dw-reduce hi lo m base)) - (d (mod (- a r) base)) - (d (if (< a r) (mod (+ d m) base) d))) - d)) - -(defthmd dw-submod-aux1 - (implies (and (natp a) (< 0 m) (natp m) - (natp x) (equal r (mod x m))) - (equal (mod (- a x) m) - (mod (- a r) m)))) - -(defthmd dw-submod-correct - (implies (and (< 0 m) (< m base) - (natp a) (< a m) - (< hi base) (< lo base) - (natp m) (natp base) - (natp hi) (natp lo)) - (equal (dw-submod a hi lo m base) - (mod (- a (+ (* base hi) lo)) m))) - :hints (("Goal" :in-theory (disable dw-reduce) - :use ((:instance dw-submod-aux1 - (x (+ lo (* base hi))) - (r (dw-reduce hi lo m base))) - (:instance dw-reduce-correct))))) - - -;; ========================================================================= -;; ANSI C arithmetic for uint64_t -;; ========================================================================= - -(defun add (a b) - (mod (+ a b) - (expt 2 64))) - -(defun sub (a b) - (mod (- a b) - (expt 2 64))) - -(defun << (w n) - (mod (* w (expt 2 n)) - (expt 2 64))) - -(defun >> (w n) - (floor w (expt 2 n))) - -;; join upper and lower half of a double word, yielding a 128 bit number -(defun join (hi lo) - (+ (* (expt 2 64) hi) lo)) - - -;; ============================================================================= -;; Fast modular reduction -;; ============================================================================= - -;; These are the three primes used in the Number Theoretic Transform. -;; A fast modular reduction scheme exists for all of them. -(defmacro p1 () - (+ (expt 2 64) (- (expt 2 32)) 1)) - -(defmacro p2 () - (+ (expt 2 64) (- (expt 2 34)) 1)) - -(defmacro p3 () - (+ (expt 2 64) (- (expt 2 40)) 1)) - - -;; reduce the double word number hi*2**64 + lo (mod p1) -(defun simple-mod-reduce-p1 (hi lo) - (+ (* (expt 2 32) hi) (- hi) lo)) - -;; reduce the double word number hi*2**64 + lo (mod p2) -(defun simple-mod-reduce-p2 (hi lo) - (+ (* (expt 2 34) hi) (- hi) lo)) - -;; reduce the double word number hi*2**64 + lo (mod p3) -(defun simple-mod-reduce-p3 (hi lo) - (+ (* (expt 2 40) hi) (- hi) lo)) - - -; ---------------------------------------------------------- -; The modular reductions given above are correct -; ---------------------------------------------------------- - -(defthmd congruence-p1-aux - (equal (* (expt 2 64) hi) - (+ (* (p1) hi) - (* (expt 2 32) hi) - (- hi)))) - -(defthmd congruence-p2-aux - (equal (* (expt 2 64) hi) - (+ (* (p2) hi) - (* (expt 2 34) hi) - (- hi)))) - -(defthmd congruence-p3-aux - (equal (* (expt 2 64) hi) - (+ (* (p3) hi) - (* (expt 2 40) hi) - (- hi)))) - -(defthmd mod-augment - (implies (and (rationalp x) - (rationalp y) - (rationalp m)) - (equal (mod (+ x y) m) - (mod (+ x (mod y m)) m)))) - -(defthmd simple-mod-reduce-p1-congruent - (implies (and (integerp hi) - (integerp lo)) - (equal (mod (simple-mod-reduce-p1 hi lo) (p1)) - (mod (join hi lo) (p1)))) - :hints (("Goal''" :use ((:instance congruence-p1-aux) - (:instance mod-augment - (m (p1)) - (x (+ (- hi) lo (* (expt 2 32) hi))) - (y (* (p1) hi))))))) - -(defthmd simple-mod-reduce-p2-congruent - (implies (and (integerp hi) - (integerp lo)) - (equal (mod (simple-mod-reduce-p2 hi lo) (p2)) - (mod (join hi lo) (p2)))) - :hints (("Goal''" :use ((:instance congruence-p2-aux) - (:instance mod-augment - (m (p2)) - (x (+ (- hi) lo (* (expt 2 34) hi))) - (y (* (p2) hi))))))) - -(defthmd simple-mod-reduce-p3-congruent - (implies (and (integerp hi) - (integerp lo)) - (equal (mod (simple-mod-reduce-p3 hi lo) (p3)) - (mod (join hi lo) (p3)))) - :hints (("Goal''" :use ((:instance congruence-p3-aux) - (:instance mod-augment - (m (p3)) - (x (+ (- hi) lo (* (expt 2 40) hi))) - (y (* (p3) hi))))))) - - -; --------------------------------------------------------------------- -; We need a number less than 2*p, so that we can use the trick from -; elim-mod-m<x<2*m for the final reduction. -; For p1, two modular reductions are sufficient, for p2 and p3 three. -; --------------------------------------------------------------------- - -;; p1: the first reduction is less than 2**96 -(defthmd simple-mod-reduce-p1-<-2**96 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (natp hi) (natp lo)) - (< (simple-mod-reduce-p1 hi lo) - (expt 2 96)))) - -;; p1: the second reduction is less than 2*p1 -(defthmd simple-mod-reduce-p1-<-2*p1 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (< (join hi lo) (expt 2 96)) - (natp hi) (natp lo)) - (< (simple-mod-reduce-p1 hi lo) - (* 2 (p1))))) - - -;; p2: the first reduction is less than 2**98 -(defthmd simple-mod-reduce-p2-<-2**98 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (natp hi) (natp lo)) - (< (simple-mod-reduce-p2 hi lo) - (expt 2 98)))) - -;; p2: the second reduction is less than 2**69 -(defthmd simple-mod-reduce-p2-<-2*69 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (< (join hi lo) (expt 2 98)) - (natp hi) (natp lo)) - (< (simple-mod-reduce-p2 hi lo) - (expt 2 69)))) - -;; p3: the third reduction is less than 2*p2 -(defthmd simple-mod-reduce-p2-<-2*p2 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (< (join hi lo) (expt 2 69)) - (natp hi) (natp lo)) - (< (simple-mod-reduce-p2 hi lo) - (* 2 (p2))))) - - -;; p3: the first reduction is less than 2**104 -(defthmd simple-mod-reduce-p3-<-2**104 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (natp hi) (natp lo)) - (< (simple-mod-reduce-p3 hi lo) - (expt 2 104)))) - -;; p3: the second reduction is less than 2**81 -(defthmd simple-mod-reduce-p3-<-2**81 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (< (join hi lo) (expt 2 104)) - (natp hi) (natp lo)) - (< (simple-mod-reduce-p3 hi lo) - (expt 2 81)))) - -;; p3: the third reduction is less than 2*p3 -(defthmd simple-mod-reduce-p3-<-2*p3 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (< (join hi lo) (expt 2 81)) - (natp hi) (natp lo)) - (< (simple-mod-reduce-p3 hi lo) - (* 2 (p3))))) - - -; ------------------------------------------------------------------------- -; The simple modular reductions, adapted for compiler friendly C -; ------------------------------------------------------------------------- - -(defun mod-reduce-p1 (hi lo) - (let* ((y hi) - (x y) - (hi (>> hi 32)) - (x (sub lo x)) - (hi (if (> x lo) (+ hi -1) hi)) - (y (<< y 32)) - (lo (add y x)) - (hi (if (< lo y) (+ hi 1) hi))) - (+ (* hi (expt 2 64)) lo))) - -(defun mod-reduce-p2 (hi lo) - (let* ((y hi) - (x y) - (hi (>> hi 30)) - (x (sub lo x)) - (hi (if (> x lo) (+ hi -1) hi)) - (y (<< y 34)) - (lo (add y x)) - (hi (if (< lo y) (+ hi 1) hi))) - (+ (* hi (expt 2 64)) lo))) - -(defun mod-reduce-p3 (hi lo) - (let* ((y hi) - (x y) - (hi (>> hi 24)) - (x (sub lo x)) - (hi (if (> x lo) (+ hi -1) hi)) - (y (<< y 40)) - (lo (add y x)) - (hi (if (< lo y) (+ hi 1) hi))) - (+ (* hi (expt 2 64)) lo))) - - -; ------------------------------------------------------------------------- -; The compiler friendly versions are equal to the simple versions -; ------------------------------------------------------------------------- - -(defthm mod-reduce-aux1 - (implies (and (<= 0 a) (natp a) (natp m) - (< (- m) b) (<= b 0) - (integerp b) - (< (mod (+ b a) m) - (mod a m))) - (equal (mod (+ b a) m) - (+ b (mod a m)))) - :hints (("Subgoal 2" :use ((:instance modaux-1b - (x (+ a b))))))) - -(defthm mod-reduce-aux2 - (implies (and (<= 0 a) (natp a) (natp m) - (< b m) (natp b) - (< (mod (+ b a) m) - (mod a m))) - (equal (+ m (mod (+ b a) m)) - (+ b (mod a m))))) - - -(defthm mod-reduce-aux3 - (implies (and (< 0 a) (natp a) (natp m) - (< (- m) b) (< b 0) - (integerp b) - (<= (mod a m) - (mod (+ b a) m))) - (equal (+ (- m) (mod (+ b a) m)) - (+ b (mod a m)))) - :hints (("Subgoal 1.2'" :use ((:instance modaux-1b - (x b)))) - ("Subgoal 1''" :use ((:instance modaux-2d - (x I)))))) - - -(defthm mod-reduce-aux4 - (implies (and (< 0 a) (natp a) (natp m) - (< b m) (natp b) - (<= (mod a m) - (mod (+ b a) m))) - (equal (mod (+ b a) m) - (+ b (mod a m))))) - - -(defthm mod-reduce-p1==simple-mod-reduce-p1 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (natp hi) (natp lo)) - (equal (mod-reduce-p1 hi lo) - (simple-mod-reduce-p1 hi lo))) - :hints (("Goal" :in-theory (disable expt) - :cases ((< 0 hi))) - ("Subgoal 1.2.2'" :use ((:instance mod-reduce-aux1 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 32) hi))))) - ("Subgoal 1.2.1'" :use ((:instance mod-reduce-aux3 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 32) hi))))) - ("Subgoal 1.1.2'" :use ((:instance mod-reduce-aux2 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 32) hi))))) - ("Subgoal 1.1.1'" :use ((:instance mod-reduce-aux4 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 32) hi))))))) - - -(defthm mod-reduce-p2==simple-mod-reduce-p2 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (natp hi) (natp lo)) - (equal (mod-reduce-p2 hi lo) - (simple-mod-reduce-p2 hi lo))) - :hints (("Goal" :cases ((< 0 hi))) - ("Subgoal 1.2.2'" :use ((:instance mod-reduce-aux1 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 34) hi))))) - ("Subgoal 1.2.1'" :use ((:instance mod-reduce-aux3 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 34) hi))))) - ("Subgoal 1.1.2'" :use ((:instance mod-reduce-aux2 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 34) hi))))) - ("Subgoal 1.1.1'" :use ((:instance mod-reduce-aux4 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 34) hi))))))) - - -(defthm mod-reduce-p3==simple-mod-reduce-p3 - (implies (and (< hi (expt 2 64)) - (< lo (expt 2 64)) - (natp hi) (natp lo)) - (equal (mod-reduce-p3 hi lo) - (simple-mod-reduce-p3 hi lo))) - :hints (("Goal" :cases ((< 0 hi))) - ("Subgoal 1.2.2'" :use ((:instance mod-reduce-aux1 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 40) hi))))) - ("Subgoal 1.2.1'" :use ((:instance mod-reduce-aux3 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 40) hi))))) - ("Subgoal 1.1.2'" :use ((:instance mod-reduce-aux2 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 40) hi))))) - ("Subgoal 1.1.1'" :use ((:instance mod-reduce-aux4 - (m (expt 2 64)) - (b (+ (- HI) LO)) - (a (* (expt 2 40) hi))))))) - - - diff --git a/Modules/_decimal/libmpdec/mpalloc.c b/Modules/_decimal/libmpdec/mpalloc.c deleted file mode 100644 index 5871d5c0f535198..000000000000000 --- a/Modules/_decimal/libmpdec/mpalloc.c +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "mpalloc.h" -#include "typearith.h" - - -#if defined(_MSC_VER) - #pragma warning(disable : 4232) -#endif - - -/* Guaranteed minimum allocation for a coefficient. May be changed once - at program start using mpd_setminalloc(). */ -mpd_ssize_t MPD_MINALLOC = MPD_MINALLOC_MIN; - -/* Custom allocation and free functions */ -void *(* mpd_mallocfunc)(size_t size) = malloc; -void *(* mpd_reallocfunc)(void *ptr, size_t size) = realloc; -void *(* mpd_callocfunc)(size_t nmemb, size_t size) = calloc; -void (* mpd_free)(void *ptr) = free; - - -/* emulate calloc if it is not available */ -void * -mpd_callocfunc_em(size_t nmemb, size_t size) -{ - void *ptr; - size_t req; - mpd_size_t overflow; - - req = mul_size_t_overflow((mpd_size_t)nmemb, (mpd_size_t)size, - &overflow); - if (overflow) { - return NULL; - } - - ptr = mpd_mallocfunc(req); - if (ptr == NULL) { - return NULL; - } - /* used on uint32_t or uint64_t */ - memset(ptr, 0, req); - - return ptr; -} - - -/* malloc with overflow checking */ -void * -mpd_alloc(mpd_size_t nmemb, mpd_size_t size) -{ - mpd_size_t req, overflow; - - req = mul_size_t_overflow(nmemb, size, &overflow); - if (overflow) { - return NULL; - } - - return mpd_mallocfunc(req); -} - -/* calloc with overflow checking */ -void * -mpd_calloc(mpd_size_t nmemb, mpd_size_t size) -{ - mpd_size_t overflow; - - (void)mul_size_t_overflow(nmemb, size, &overflow); - if (overflow) { - return NULL; - } - - return mpd_callocfunc(nmemb, size); -} - -/* realloc with overflow checking */ -void * -mpd_realloc(void *ptr, mpd_size_t nmemb, mpd_size_t size, uint8_t *err) -{ - void *new; - mpd_size_t req, overflow; - - req = mul_size_t_overflow(nmemb, size, &overflow); - if (overflow) { - *err = 1; - return ptr; - } - - new = mpd_reallocfunc(ptr, req); - if (new == NULL) { - *err = 1; - return ptr; - } - - return new; -} - -/* struct hack malloc with overflow checking */ -void * -mpd_sh_alloc(mpd_size_t struct_size, mpd_size_t nmemb, mpd_size_t size) -{ - mpd_size_t req, overflow; - - req = mul_size_t_overflow(nmemb, size, &overflow); - if (overflow) { - return NULL; - } - - req = add_size_t_overflow(req, struct_size, &overflow); - if (overflow) { - return NULL; - } - - return mpd_mallocfunc(req); -} - - -/* Allocate a new decimal with a coefficient of length 'nwords'. In case - of an error the return value is NULL. */ -mpd_t * -mpd_qnew_size(mpd_ssize_t nwords) -{ - mpd_t *result; - - nwords = (nwords < MPD_MINALLOC) ? MPD_MINALLOC : nwords; - - result = mpd_alloc(1, sizeof *result); - if (result == NULL) { - return NULL; - } - - result->data = mpd_alloc(nwords, sizeof *result->data); - if (result->data == NULL) { - mpd_free(result); - return NULL; - } - - result->flags = 0; - result->exp = 0; - result->digits = 0; - result->len = 0; - result->alloc = nwords; - - return result; -} - -/* Allocate a new decimal with a coefficient of length MPD_MINALLOC. - In case of an error the return value is NULL. */ -mpd_t * -mpd_qnew(void) -{ - return mpd_qnew_size(MPD_MINALLOC); -} - -/* Allocate new decimal. Caller can check for NULL or MPD_Malloc_error. - Raises on error. */ -mpd_t * -mpd_new(mpd_context_t *ctx) -{ - mpd_t *result; - - result = mpd_qnew(); - if (result == NULL) { - mpd_addstatus_raise(ctx, MPD_Malloc_error); - } - return result; -} - -/* - * Input: 'result' is a static mpd_t with a static coefficient. - * Assumption: 'nwords' >= result->alloc. - * - * Resize the static coefficient to a larger dynamic one and copy the - * existing data. If successful, the value of 'result' is unchanged. - * Otherwise, set 'result' to NaN and update 'status' with MPD_Malloc_error. - */ -int -mpd_switch_to_dyn(mpd_t *result, mpd_ssize_t nwords, uint32_t *status) -{ - mpd_uint_t *p = result->data; - - assert(nwords >= result->alloc); - - result->data = mpd_alloc(nwords, sizeof *result->data); - if (result->data == NULL) { - result->data = p; - mpd_set_qnan(result); - mpd_set_positive(result); - result->exp = result->digits = result->len = 0; - *status |= MPD_Malloc_error; - return 0; - } - - memcpy(result->data, p, result->alloc * (sizeof *result->data)); - result->alloc = nwords; - mpd_set_dynamic_data(result); - return 1; -} - -/* - * Input: 'result' is a static mpd_t with a static coefficient. - * - * Convert the coefficient to a dynamic one that is initialized to zero. If - * malloc fails, set 'result' to NaN and update 'status' with MPD_Malloc_error. - */ -int -mpd_switch_to_dyn_zero(mpd_t *result, mpd_ssize_t nwords, uint32_t *status) -{ - mpd_uint_t *p = result->data; - - result->data = mpd_calloc(nwords, sizeof *result->data); - if (result->data == NULL) { - result->data = p; - mpd_set_qnan(result); - mpd_set_positive(result); - result->exp = result->digits = result->len = 0; - *status |= MPD_Malloc_error; - return 0; - } - - result->alloc = nwords; - mpd_set_dynamic_data(result); - - return 1; -} - -/* - * Input: 'result' is a static or a dynamic mpd_t with a dynamic coefficient. - * Resize the coefficient to length 'nwords': - * Case nwords > result->alloc: - * If realloc is successful: - * 'result' has a larger coefficient but the same value. Return 1. - * Otherwise: - * Set 'result' to NaN, update status with MPD_Malloc_error and return 0. - * Case nwords < result->alloc: - * If realloc is successful: - * 'result' has a smaller coefficient. result->len is undefined. Return 1. - * Otherwise (unlikely): - * 'result' is unchanged. Reuse the now oversized coefficient. Return 1. - */ -int -mpd_realloc_dyn(mpd_t *result, mpd_ssize_t nwords, uint32_t *status) -{ - uint8_t err = 0; - - result->data = mpd_realloc(result->data, nwords, sizeof *result->data, &err); - if (!err) { - result->alloc = nwords; - } - else if (nwords > result->alloc) { - mpd_set_qnan(result); - mpd_set_positive(result); - result->exp = result->digits = result->len = 0; - *status |= MPD_Malloc_error; - return 0; - } - - return 1; -} - -/* - * Input: 'result' is a static mpd_t with a static coefficient. - * Assumption: 'nwords' >= result->alloc. - * - * Resize the static coefficient to a larger dynamic one and copy the - * existing data. - * - * On failure the value of 'result' is unchanged. - */ -int -mpd_switch_to_dyn_cxx(mpd_t *result, mpd_ssize_t nwords) -{ - assert(nwords >= result->alloc); - - mpd_uint_t *data = mpd_alloc(nwords, sizeof *result->data); - if (data == NULL) { - return 0; - } - - memcpy(data, result->data, result->alloc * (sizeof *result->data)); - result->data = data; - result->alloc = nwords; - mpd_set_dynamic_data(result); - return 1; -} - -/* - * Input: 'result' is a static or a dynamic mpd_t with a dynamic coefficient. - * Resize the coefficient to length 'nwords': - * Case nwords > result->alloc: - * If realloc is successful: - * 'result' has a larger coefficient but the same value. Return 1. - * Otherwise: - * 'result' has a the same coefficient. Return 0. - * Case nwords < result->alloc: - * If realloc is successful: - * 'result' has a smaller coefficient. result->len is undefined. Return 1. - * Otherwise (unlikely): - * 'result' is unchanged. Reuse the now oversized coefficient. Return 1. - */ -int -mpd_realloc_dyn_cxx(mpd_t *result, mpd_ssize_t nwords) -{ - uint8_t err = 0; - - mpd_uint_t *p = mpd_realloc(result->data, nwords, sizeof *result->data, &err); - if (!err) { - result->data = p; - result->alloc = nwords; - } - else if (nwords > result->alloc) { - return 0; - } - - return 1; -} diff --git a/Modules/_decimal/libmpdec/mpalloc.h b/Modules/_decimal/libmpdec/mpalloc.h deleted file mode 100644 index 226500442182413..000000000000000 --- a/Modules/_decimal/libmpdec/mpalloc.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_MPALLOC_H_ -#define LIBMPDEC_MPALLOC_H_ - - -#include "mpdecimal.h" - -#include <stdint.h> - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -int mpd_switch_to_dyn(mpd_t *result, mpd_ssize_t nwords, uint32_t *status); -int mpd_switch_to_dyn_zero(mpd_t *result, mpd_ssize_t nwords, uint32_t *status); -int mpd_realloc_dyn(mpd_t *result, mpd_ssize_t nwords, uint32_t *status); - -int mpd_switch_to_dyn_cxx(mpd_t *result, mpd_ssize_t nwords); -int mpd_realloc_dyn_cxx(mpd_t *result, mpd_ssize_t nwords); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_MPALLOC_H_ */ diff --git a/Modules/_decimal/libmpdec/mpdecimal.c b/Modules/_decimal/libmpdec/mpdecimal.c deleted file mode 100644 index 959934bda7a449b..000000000000000 --- a/Modules/_decimal/libmpdec/mpdecimal.c +++ /dev/null @@ -1,9015 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> -#include <limits.h> -#include <math.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "basearith.h" -#include "bits.h" -#include "constants.h" -#include "convolute.h" -#include "crt.h" -#include "mpalloc.h" -#include "typearith.h" - -#ifdef PPRO - #if defined(_MSC_VER) - #include <float.h> - #pragma float_control(precise, on) - #pragma fenv_access(on) - #elif !defined(__OpenBSD__) && !defined(__NetBSD__) - /* C99 */ - #include <fenv.h> - #pragma STDC FENV_ACCESS ON - #endif -#endif - - -/* Disable warning that is part of -Wextra since gcc 7.0. */ -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && __GNUC__ >= 7 - #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" -#endif - - -#if defined(_MSC_VER) - #define ALWAYS_INLINE __forceinline -#elif defined (__IBMC__) || defined(LEGACY_COMPILER) - #define ALWAYS_INLINE - #undef inline - #define inline -#else - #ifdef TEST_COVERAGE - #define ALWAYS_INLINE - #else - #define ALWAYS_INLINE inline __attribute__ ((always_inline)) - #endif -#endif - -/* ClangCL claims to support 128-bit int, but doesn't */ -#if defined(__SIZEOF_INT128__) && defined(__clang__) && defined(_MSC_VER) -#undef __SIZEOF_INT128__ -#endif - - - -#define MPD_NEWTONDIV_CUTOFF 1024L - -#define MPD_NEW_STATIC(name, flags, exp, digits, len) \ - mpd_uint_t name##_data[MPD_MINALLOC_MAX]; \ - mpd_t name = {flags|MPD_STATIC|MPD_STATIC_DATA, exp, digits, \ - len, MPD_MINALLOC_MAX, name##_data} - -#define MPD_NEW_CONST(name, flags, exp, digits, len, alloc, initval) \ - mpd_uint_t name##_data[alloc] = {initval}; \ - mpd_t name = {flags|MPD_STATIC|MPD_CONST_DATA, exp, digits, \ - len, alloc, name##_data} - -#define MPD_NEW_SHARED(name, a) \ - mpd_t name = {(a->flags&~MPD_DATAFLAGS)|MPD_STATIC|MPD_SHARED_DATA, \ - a->exp, a->digits, a->len, a->alloc, a->data} - - -static mpd_uint_t data_one[1] = {1}; -static mpd_uint_t data_zero[1] = {0}; -static const mpd_t one = {MPD_STATIC|MPD_CONST_DATA, 0, 1, 1, 1, data_one}; -static const mpd_t minus_one = {MPD_NEG|MPD_STATIC|MPD_CONST_DATA, 0, 1, 1, 1, - data_one}; -static const mpd_t zero = {MPD_STATIC|MPD_CONST_DATA, 0, 1, 1, 1, data_zero}; - -static inline void _mpd_check_exp(mpd_t *dec, const mpd_context_t *ctx, - uint32_t *status); -static void _settriple(mpd_t *result, uint8_t sign, mpd_uint_t a, - mpd_ssize_t exp); -static inline mpd_ssize_t _mpd_real_size(mpd_uint_t *data, mpd_ssize_t size); - -static int _mpd_cmp_abs(const mpd_t *a, const mpd_t *b); - -static void _mpd_qadd(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status); -static inline void _mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status); -static void _mpd_base_ndivmod(mpd_t *q, mpd_t *r, const mpd_t *a, - const mpd_t *b, uint32_t *status); -static inline void _mpd_qpow_uint(mpd_t *result, const mpd_t *base, - mpd_uint_t exp, uint8_t resultsign, - const mpd_context_t *ctx, uint32_t *status); - -static mpd_uint_t mpd_qsshiftr(mpd_t *result, const mpd_t *a, mpd_ssize_t n); - - -/******************************************************************************/ -/* Version */ -/******************************************************************************/ - -const char * -mpd_version(void) -{ - return MPD_VERSION; -} - - -/******************************************************************************/ -/* Performance critical inline functions */ -/******************************************************************************/ - -#ifdef CONFIG_64 -/* Digits in a word, primarily useful for the most significant word. */ -ALWAYS_INLINE int -mpd_word_digits(mpd_uint_t word) -{ - if (word < mpd_pow10[9]) { - if (word < mpd_pow10[4]) { - if (word < mpd_pow10[2]) { - return (word < mpd_pow10[1]) ? 1 : 2; - } - return (word < mpd_pow10[3]) ? 3 : 4; - } - if (word < mpd_pow10[6]) { - return (word < mpd_pow10[5]) ? 5 : 6; - } - if (word < mpd_pow10[8]) { - return (word < mpd_pow10[7]) ? 7 : 8; - } - return 9; - } - if (word < mpd_pow10[14]) { - if (word < mpd_pow10[11]) { - return (word < mpd_pow10[10]) ? 10 : 11; - } - if (word < mpd_pow10[13]) { - return (word < mpd_pow10[12]) ? 12 : 13; - } - return 14; - } - if (word < mpd_pow10[18]) { - if (word < mpd_pow10[16]) { - return (word < mpd_pow10[15]) ? 15 : 16; - } - return (word < mpd_pow10[17]) ? 17 : 18; - } - - return (word < mpd_pow10[19]) ? 19 : 20; -} -#else -ALWAYS_INLINE int -mpd_word_digits(mpd_uint_t word) -{ - if (word < mpd_pow10[4]) { - if (word < mpd_pow10[2]) { - return (word < mpd_pow10[1]) ? 1 : 2; - } - return (word < mpd_pow10[3]) ? 3 : 4; - } - if (word < mpd_pow10[6]) { - return (word < mpd_pow10[5]) ? 5 : 6; - } - if (word < mpd_pow10[8]) { - return (word < mpd_pow10[7]) ? 7 : 8; - } - - return (word < mpd_pow10[9]) ? 9 : 10; -} -#endif - - -/* Adjusted exponent */ -ALWAYS_INLINE mpd_ssize_t -mpd_adjexp(const mpd_t *dec) -{ - return (dec->exp + dec->digits) - 1; -} - -/* Etiny */ -ALWAYS_INLINE mpd_ssize_t -mpd_etiny(const mpd_context_t *ctx) -{ - return ctx->emin - (ctx->prec - 1); -} - -/* Etop: used for folding down in IEEE clamping */ -ALWAYS_INLINE mpd_ssize_t -mpd_etop(const mpd_context_t *ctx) -{ - return ctx->emax - (ctx->prec - 1); -} - -/* Most significant word */ -ALWAYS_INLINE mpd_uint_t -mpd_msword(const mpd_t *dec) -{ - assert(dec->len > 0); - return dec->data[dec->len-1]; -} - -/* Most significant digit of a word */ -inline mpd_uint_t -mpd_msd(mpd_uint_t word) -{ - int n; - - n = mpd_word_digits(word); - return word / mpd_pow10[n-1]; -} - -/* Least significant digit of a word */ -ALWAYS_INLINE mpd_uint_t -mpd_lsd(mpd_uint_t word) -{ - return word % 10; -} - -/* Coefficient size needed to store 'digits' */ -mpd_ssize_t -mpd_digits_to_size(mpd_ssize_t digits) -{ - mpd_ssize_t q, r; - - _mpd_idiv_word(&q, &r, digits, MPD_RDIGITS); - return (r == 0) ? q : q+1; -} - -/* Number of digits in the exponent. Not defined for MPD_SSIZE_MIN. */ -inline int -mpd_exp_digits(mpd_ssize_t exp) -{ - exp = (exp < 0) ? -exp : exp; - return mpd_word_digits(exp); -} - -/* Canonical */ -ALWAYS_INLINE int -mpd_iscanonical(const mpd_t *dec) -{ - (void)dec; - return 1; -} - -/* Finite */ -ALWAYS_INLINE int -mpd_isfinite(const mpd_t *dec) -{ - return !(dec->flags & MPD_SPECIAL); -} - -/* Infinite */ -ALWAYS_INLINE int -mpd_isinfinite(const mpd_t *dec) -{ - return dec->flags & MPD_INF; -} - -/* NaN */ -ALWAYS_INLINE int -mpd_isnan(const mpd_t *dec) -{ - return dec->flags & (MPD_NAN|MPD_SNAN); -} - -/* Negative */ -ALWAYS_INLINE int -mpd_isnegative(const mpd_t *dec) -{ - return dec->flags & MPD_NEG; -} - -/* Positive */ -ALWAYS_INLINE int -mpd_ispositive(const mpd_t *dec) -{ - return !(dec->flags & MPD_NEG); -} - -/* qNaN */ -ALWAYS_INLINE int -mpd_isqnan(const mpd_t *dec) -{ - return dec->flags & MPD_NAN; -} - -/* Signed */ -ALWAYS_INLINE int -mpd_issigned(const mpd_t *dec) -{ - return dec->flags & MPD_NEG; -} - -/* sNaN */ -ALWAYS_INLINE int -mpd_issnan(const mpd_t *dec) -{ - return dec->flags & MPD_SNAN; -} - -/* Special */ -ALWAYS_INLINE int -mpd_isspecial(const mpd_t *dec) -{ - return dec->flags & MPD_SPECIAL; -} - -/* Zero */ -ALWAYS_INLINE int -mpd_iszero(const mpd_t *dec) -{ - return !mpd_isspecial(dec) && mpd_msword(dec) == 0; -} - -/* Test for zero when specials have been ruled out already */ -ALWAYS_INLINE int -mpd_iszerocoeff(const mpd_t *dec) -{ - return mpd_msword(dec) == 0; -} - -/* Normal */ -inline int -mpd_isnormal(const mpd_t *dec, const mpd_context_t *ctx) -{ - if (mpd_isspecial(dec)) return 0; - if (mpd_iszerocoeff(dec)) return 0; - - return mpd_adjexp(dec) >= ctx->emin; -} - -/* Subnormal */ -inline int -mpd_issubnormal(const mpd_t *dec, const mpd_context_t *ctx) -{ - if (mpd_isspecial(dec)) return 0; - if (mpd_iszerocoeff(dec)) return 0; - - return mpd_adjexp(dec) < ctx->emin; -} - -/* Odd word */ -ALWAYS_INLINE int -mpd_isoddword(mpd_uint_t word) -{ - return word & 1; -} - -/* Odd coefficient */ -ALWAYS_INLINE int -mpd_isoddcoeff(const mpd_t *dec) -{ - return mpd_isoddword(dec->data[0]); -} - -/* 0 if dec is positive, 1 if dec is negative */ -ALWAYS_INLINE uint8_t -mpd_sign(const mpd_t *dec) -{ - return dec->flags & MPD_NEG; -} - -/* 1 if dec is positive, -1 if dec is negative */ -ALWAYS_INLINE int -mpd_arith_sign(const mpd_t *dec) -{ - return 1 - 2 * mpd_isnegative(dec); -} - -/* Radix */ -ALWAYS_INLINE long -mpd_radix(void) -{ - return 10; -} - -/* Dynamic decimal */ -ALWAYS_INLINE int -mpd_isdynamic(const mpd_t *dec) -{ - return !(dec->flags & MPD_STATIC); -} - -/* Static decimal */ -ALWAYS_INLINE int -mpd_isstatic(const mpd_t *dec) -{ - return dec->flags & MPD_STATIC; -} - -/* Data of decimal is dynamic */ -ALWAYS_INLINE int -mpd_isdynamic_data(const mpd_t *dec) -{ - return !(dec->flags & MPD_DATAFLAGS); -} - -/* Data of decimal is static */ -ALWAYS_INLINE int -mpd_isstatic_data(const mpd_t *dec) -{ - return dec->flags & MPD_STATIC_DATA; -} - -/* Data of decimal is shared */ -ALWAYS_INLINE int -mpd_isshared_data(const mpd_t *dec) -{ - return dec->flags & MPD_SHARED_DATA; -} - -/* Data of decimal is const */ -ALWAYS_INLINE int -mpd_isconst_data(const mpd_t *dec) -{ - return dec->flags & MPD_CONST_DATA; -} - - -/******************************************************************************/ -/* Inline memory handling */ -/******************************************************************************/ - -/* Fill destination with zeros */ -ALWAYS_INLINE void -mpd_uint_zero(mpd_uint_t *dest, mpd_size_t len) -{ - mpd_size_t i; - - for (i = 0; i < len; i++) { - dest[i] = 0; - } -} - -/* Free a decimal */ -ALWAYS_INLINE void -mpd_del(mpd_t *dec) -{ - if (mpd_isdynamic_data(dec)) { - mpd_free(dec->data); - } - if (mpd_isdynamic(dec)) { - mpd_free(dec); - } -} - -/* - * Resize the coefficient. Existing data up to 'nwords' is left untouched. - * Return 1 on success, 0 otherwise. - * - * Input invariant: MPD_MINALLOC <= result->alloc. - * - * Case nwords == result->alloc: - * 'result' is unchanged. Return 1. - * - * Case nwords > result->alloc: - * Case realloc success: - * The value of 'result' does not change. Return 1. - * Case realloc failure: - * 'result' is NaN, status is updated with MPD_Malloc_error. Return 0. - * - * Case nwords < result->alloc: - * Case is_static_data or realloc failure [1]: - * 'result' is unchanged. Return 1. - * Case realloc success: - * The value of result is undefined (expected). Return 1. - * - * - * [1] In that case the old (now oversized) area is still valid. - */ -ALWAYS_INLINE int -mpd_qresize(mpd_t *result, mpd_ssize_t nwords, uint32_t *status) -{ - assert(!mpd_isconst_data(result)); /* illegal operation for a const */ - assert(!mpd_isshared_data(result)); /* illegal operation for a shared */ - assert(MPD_MINALLOC <= result->alloc); - - nwords = (nwords <= MPD_MINALLOC) ? MPD_MINALLOC : nwords; - if (nwords == result->alloc) { - return 1; - } - if (mpd_isstatic_data(result)) { - if (nwords > result->alloc) { - return mpd_switch_to_dyn(result, nwords, status); - } - return 1; - } - - return mpd_realloc_dyn(result, nwords, status); -} - -/* Same as mpd_qresize, but do not set the result no NaN on failure. */ -static ALWAYS_INLINE int -mpd_qresize_cxx(mpd_t *result, mpd_ssize_t nwords) -{ - assert(!mpd_isconst_data(result)); /* illegal operation for a const */ - assert(!mpd_isshared_data(result)); /* illegal operation for a shared */ - assert(MPD_MINALLOC <= result->alloc); - - nwords = (nwords <= MPD_MINALLOC) ? MPD_MINALLOC : nwords; - if (nwords == result->alloc) { - return 1; - } - if (mpd_isstatic_data(result)) { - if (nwords > result->alloc) { - return mpd_switch_to_dyn_cxx(result, nwords); - } - return 1; - } - - return mpd_realloc_dyn_cxx(result, nwords); -} - -/* Same as mpd_qresize, but the complete coefficient (including the old - * memory area!) is initialized to zero. */ -ALWAYS_INLINE int -mpd_qresize_zero(mpd_t *result, mpd_ssize_t nwords, uint32_t *status) -{ - assert(!mpd_isconst_data(result)); /* illegal operation for a const */ - assert(!mpd_isshared_data(result)); /* illegal operation for a shared */ - assert(MPD_MINALLOC <= result->alloc); - - nwords = (nwords <= MPD_MINALLOC) ? MPD_MINALLOC : nwords; - if (nwords != result->alloc) { - if (mpd_isstatic_data(result)) { - if (nwords > result->alloc) { - return mpd_switch_to_dyn_zero(result, nwords, status); - } - } - else if (!mpd_realloc_dyn(result, nwords, status)) { - return 0; - } - } - - mpd_uint_zero(result->data, nwords); - return 1; -} - -/* - * Reduce memory size for the coefficient to MPD_MINALLOC. In theory, - * realloc may fail even when reducing the memory size. But in that case - * the old memory area is always big enough, so checking for MPD_Malloc_error - * is not imperative. - */ -ALWAYS_INLINE void -mpd_minalloc(mpd_t *result) -{ - assert(!mpd_isconst_data(result)); /* illegal operation for a const */ - assert(!mpd_isshared_data(result)); /* illegal operation for a shared */ - - if (!mpd_isstatic_data(result) && result->alloc > MPD_MINALLOC) { - uint8_t err = 0; - result->data = mpd_realloc(result->data, MPD_MINALLOC, - sizeof *result->data, &err); - if (!err) { - result->alloc = MPD_MINALLOC; - } - } -} - -int -mpd_resize(mpd_t *result, mpd_ssize_t nwords, mpd_context_t *ctx) -{ - uint32_t status = 0; - if (!mpd_qresize(result, nwords, &status)) { - mpd_addstatus_raise(ctx, status); - return 0; - } - return 1; -} - -int -mpd_resize_zero(mpd_t *result, mpd_ssize_t nwords, mpd_context_t *ctx) -{ - uint32_t status = 0; - if (!mpd_qresize_zero(result, nwords, &status)) { - mpd_addstatus_raise(ctx, status); - return 0; - } - return 1; -} - - -/******************************************************************************/ -/* Set attributes of a decimal */ -/******************************************************************************/ - -/* Set digits. Assumption: result->len is initialized and > 0. */ -inline void -mpd_setdigits(mpd_t *result) -{ - mpd_ssize_t wdigits = mpd_word_digits(mpd_msword(result)); - result->digits = wdigits + (result->len-1) * MPD_RDIGITS; -} - -/* Set sign */ -ALWAYS_INLINE void -mpd_set_sign(mpd_t *result, uint8_t sign) -{ - result->flags &= ~MPD_NEG; - result->flags |= sign; -} - -/* Copy sign from another decimal */ -ALWAYS_INLINE void -mpd_signcpy(mpd_t *result, const mpd_t *a) -{ - uint8_t sign = a->flags&MPD_NEG; - - result->flags &= ~MPD_NEG; - result->flags |= sign; -} - -/* Set infinity */ -ALWAYS_INLINE void -mpd_set_infinity(mpd_t *result) -{ - result->flags &= ~MPD_SPECIAL; - result->flags |= MPD_INF; -} - -/* Set qNaN */ -ALWAYS_INLINE void -mpd_set_qnan(mpd_t *result) -{ - result->flags &= ~MPD_SPECIAL; - result->flags |= MPD_NAN; -} - -/* Set sNaN */ -ALWAYS_INLINE void -mpd_set_snan(mpd_t *result) -{ - result->flags &= ~MPD_SPECIAL; - result->flags |= MPD_SNAN; -} - -/* Set to negative */ -ALWAYS_INLINE void -mpd_set_negative(mpd_t *result) -{ - result->flags |= MPD_NEG; -} - -/* Set to positive */ -ALWAYS_INLINE void -mpd_set_positive(mpd_t *result) -{ - result->flags &= ~MPD_NEG; -} - -/* Set to dynamic */ -ALWAYS_INLINE void -mpd_set_dynamic(mpd_t *result) -{ - result->flags &= ~MPD_STATIC; -} - -/* Set to static */ -ALWAYS_INLINE void -mpd_set_static(mpd_t *result) -{ - result->flags |= MPD_STATIC; -} - -/* Set data to dynamic */ -ALWAYS_INLINE void -mpd_set_dynamic_data(mpd_t *result) -{ - result->flags &= ~MPD_DATAFLAGS; -} - -/* Set data to static */ -ALWAYS_INLINE void -mpd_set_static_data(mpd_t *result) -{ - result->flags &= ~MPD_DATAFLAGS; - result->flags |= MPD_STATIC_DATA; -} - -/* Set data to shared */ -ALWAYS_INLINE void -mpd_set_shared_data(mpd_t *result) -{ - result->flags &= ~MPD_DATAFLAGS; - result->flags |= MPD_SHARED_DATA; -} - -/* Set data to const */ -ALWAYS_INLINE void -mpd_set_const_data(mpd_t *result) -{ - result->flags &= ~MPD_DATAFLAGS; - result->flags |= MPD_CONST_DATA; -} - -/* Clear flags, preserving memory attributes. */ -ALWAYS_INLINE void -mpd_clear_flags(mpd_t *result) -{ - result->flags &= (MPD_STATIC|MPD_DATAFLAGS); -} - -/* Set flags, preserving memory attributes. */ -ALWAYS_INLINE void -mpd_set_flags(mpd_t *result, uint8_t flags) -{ - result->flags &= (MPD_STATIC|MPD_DATAFLAGS); - result->flags |= flags; -} - -/* Copy flags, preserving memory attributes of result. */ -ALWAYS_INLINE void -mpd_copy_flags(mpd_t *result, const mpd_t *a) -{ - uint8_t aflags = a->flags; - result->flags &= (MPD_STATIC|MPD_DATAFLAGS); - result->flags |= (aflags & ~(MPD_STATIC|MPD_DATAFLAGS)); -} - -/* Initialize a workcontext from ctx. Set traps, flags and newtrap to 0. */ -static inline void -mpd_workcontext(mpd_context_t *workctx, const mpd_context_t *ctx) -{ - workctx->prec = ctx->prec; - workctx->emax = ctx->emax; - workctx->emin = ctx->emin; - workctx->round = ctx->round; - workctx->traps = 0; - workctx->status = 0; - workctx->newtrap = 0; - workctx->clamp = ctx->clamp; - workctx->allcr = ctx->allcr; -} - - -/******************************************************************************/ -/* Getting and setting parts of decimals */ -/******************************************************************************/ - -/* Flip the sign of a decimal */ -static inline void -_mpd_negate(mpd_t *dec) -{ - dec->flags ^= MPD_NEG; -} - -/* Set coefficient to zero */ -void -mpd_zerocoeff(mpd_t *result) -{ - mpd_minalloc(result); - result->digits = 1; - result->len = 1; - result->data[0] = 0; -} - -/* Set the coefficient to all nines. */ -void -mpd_qmaxcoeff(mpd_t *result, const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t len, r; - - _mpd_idiv_word(&len, &r, ctx->prec, MPD_RDIGITS); - len = (r == 0) ? len : len+1; - - if (!mpd_qresize(result, len, status)) { - return; - } - - result->len = len; - result->digits = ctx->prec; - - --len; - if (r > 0) { - result->data[len--] = mpd_pow10[r]-1; - } - for (; len >= 0; --len) { - result->data[len] = MPD_RADIX-1; - } -} - -/* - * Cut off the most significant digits so that the rest fits in ctx->prec. - * Cannot fail. - */ -static void -_mpd_cap(mpd_t *result, const mpd_context_t *ctx) -{ - uint32_t dummy; - mpd_ssize_t len, r; - - if (result->len > 0 && result->digits > ctx->prec) { - _mpd_idiv_word(&len, &r, ctx->prec, MPD_RDIGITS); - len = (r == 0) ? len : len+1; - - if (r != 0) { - result->data[len-1] %= mpd_pow10[r]; - } - - len = _mpd_real_size(result->data, len); - /* resize to fewer words cannot fail */ - mpd_qresize(result, len, &dummy); - result->len = len; - mpd_setdigits(result); - } - if (mpd_iszero(result)) { - _settriple(result, mpd_sign(result), 0, result->exp); - } -} - -/* - * Cut off the most significant digits of a NaN payload so that the rest - * fits in ctx->prec - ctx->clamp. Cannot fail. - */ -static void -_mpd_fix_nan(mpd_t *result, const mpd_context_t *ctx) -{ - uint32_t dummy; - mpd_ssize_t prec; - mpd_ssize_t len, r; - - prec = ctx->prec - ctx->clamp; - if (result->len > 0 && result->digits > prec) { - if (prec == 0) { - mpd_minalloc(result); - result->len = result->digits = 0; - } - else { - _mpd_idiv_word(&len, &r, prec, MPD_RDIGITS); - len = (r == 0) ? len : len+1; - - if (r != 0) { - result->data[len-1] %= mpd_pow10[r]; - } - - len = _mpd_real_size(result->data, len); - /* resize to fewer words cannot fail */ - mpd_qresize(result, len, &dummy); - result->len = len; - mpd_setdigits(result); - if (mpd_iszerocoeff(result)) { - /* NaN0 is not a valid representation */ - result->len = result->digits = 0; - } - } - } -} - -/* - * Get n most significant digits from a decimal, where 0 < n <= MPD_UINT_DIGITS. - * Assumes MPD_UINT_DIGITS == MPD_RDIGITS+1, which is true for 32 and 64 bit - * machines. - * - * The result of the operation will be in lo. If the operation is impossible, - * hi will be nonzero. This is used to indicate an error. - */ -static inline void -_mpd_get_msdigits(mpd_uint_t *hi, mpd_uint_t *lo, const mpd_t *dec, - unsigned int n) -{ - mpd_uint_t r, tmp; - - assert(0 < n && n <= MPD_RDIGITS+1); - - _mpd_div_word(&tmp, &r, dec->digits, MPD_RDIGITS); - r = (r == 0) ? MPD_RDIGITS : r; /* digits in the most significant word */ - - *hi = 0; - *lo = dec->data[dec->len-1]; - if (n <= r) { - *lo /= mpd_pow10[r-n]; - } - else if (dec->len > 1) { - /* at this point 1 <= r < n <= MPD_RDIGITS+1 */ - _mpd_mul_words(hi, lo, *lo, mpd_pow10[n-r]); - tmp = dec->data[dec->len-2] / mpd_pow10[MPD_RDIGITS-(n-r)]; - *lo = *lo + tmp; - if (*lo < tmp) (*hi)++; - } -} - - -/******************************************************************************/ -/* Gathering information about a decimal */ -/******************************************************************************/ - -/* The real size of the coefficient without leading zero words. */ -static inline mpd_ssize_t -_mpd_real_size(mpd_uint_t *data, mpd_ssize_t size) -{ - while (size > 1 && data[size-1] == 0) { - size--; - } - - return size; -} - -/* Return number of trailing zeros. No errors are possible. */ -mpd_ssize_t -mpd_trail_zeros(const mpd_t *dec) -{ - mpd_uint_t word; - mpd_ssize_t i, tz = 0; - - for (i=0; i < dec->len; ++i) { - if (dec->data[i] != 0) { - word = dec->data[i]; - tz = i * MPD_RDIGITS; - while (word % 10 == 0) { - word /= 10; - tz++; - } - break; - } - } - - return tz; -} - -/* Integer: Undefined for specials */ -static int -_mpd_isint(const mpd_t *dec) -{ - mpd_ssize_t tz; - - if (mpd_iszerocoeff(dec)) { - return 1; - } - - tz = mpd_trail_zeros(dec); - return (dec->exp + tz >= 0); -} - -/* Integer */ -int -mpd_isinteger(const mpd_t *dec) -{ - if (mpd_isspecial(dec)) { - return 0; - } - return _mpd_isint(dec); -} - -/* Word is a power of 10 */ -static int -mpd_word_ispow10(mpd_uint_t word) -{ - int n; - - n = mpd_word_digits(word); - if (word == mpd_pow10[n-1]) { - return 1; - } - - return 0; -} - -/* Coefficient is a power of 10 */ -static int -mpd_coeff_ispow10(const mpd_t *dec) -{ - if (mpd_word_ispow10(mpd_msword(dec))) { - if (_mpd_isallzero(dec->data, dec->len-1)) { - return 1; - } - } - - return 0; -} - -/* All digits of a word are nines */ -static int -mpd_word_isallnine(mpd_uint_t word) -{ - int n; - - n = mpd_word_digits(word); - if (word == mpd_pow10[n]-1) { - return 1; - } - - return 0; -} - -/* All digits of the coefficient are nines */ -static int -mpd_coeff_isallnine(const mpd_t *dec) -{ - if (mpd_word_isallnine(mpd_msword(dec))) { - if (_mpd_isallnine(dec->data, dec->len-1)) { - return 1; - } - } - - return 0; -} - -/* Odd decimal: Undefined for non-integers! */ -int -mpd_isodd(const mpd_t *dec) -{ - mpd_uint_t q, r; - assert(mpd_isinteger(dec)); - if (mpd_iszerocoeff(dec)) return 0; - if (dec->exp < 0) { - _mpd_div_word(&q, &r, -dec->exp, MPD_RDIGITS); - q = dec->data[q] / mpd_pow10[r]; - return mpd_isoddword(q); - } - return dec->exp == 0 && mpd_isoddword(dec->data[0]); -} - -/* Even: Undefined for non-integers! */ -int -mpd_iseven(const mpd_t *dec) -{ - return !mpd_isodd(dec); -} - -/******************************************************************************/ -/* Getting and setting decimals */ -/******************************************************************************/ - -/* Internal function: Set a static decimal from a triple, no error checking. */ -static void -_ssettriple(mpd_t *result, uint8_t sign, mpd_uint_t a, mpd_ssize_t exp) -{ - mpd_set_flags(result, sign); - result->exp = exp; - _mpd_div_word(&result->data[1], &result->data[0], a, MPD_RADIX); - result->len = (result->data[1] == 0) ? 1 : 2; - mpd_setdigits(result); -} - -/* Internal function: Set a decimal from a triple, no error checking. */ -static void -_settriple(mpd_t *result, uint8_t sign, mpd_uint_t a, mpd_ssize_t exp) -{ - mpd_minalloc(result); - mpd_set_flags(result, sign); - result->exp = exp; - _mpd_div_word(&result->data[1], &result->data[0], a, MPD_RADIX); - result->len = (result->data[1] == 0) ? 1 : 2; - mpd_setdigits(result); -} - -/* Set a special number from a triple */ -void -mpd_setspecial(mpd_t *result, uint8_t sign, uint8_t type) -{ - mpd_minalloc(result); - result->flags &= ~(MPD_NEG|MPD_SPECIAL); - result->flags |= (sign|type); - result->exp = result->digits = result->len = 0; -} - -/* Set result of NaN with an error status */ -void -mpd_seterror(mpd_t *result, uint32_t flags, uint32_t *status) -{ - mpd_minalloc(result); - mpd_set_qnan(result); - mpd_set_positive(result); - result->exp = result->digits = result->len = 0; - *status |= flags; -} - -/* quietly set a static decimal from an mpd_ssize_t */ -void -mpd_qsset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_uint_t u; - uint8_t sign = MPD_POS; - - if (a < 0) { - if (a == MPD_SSIZE_MIN) { - u = (mpd_uint_t)MPD_SSIZE_MAX + - (-(MPD_SSIZE_MIN+MPD_SSIZE_MAX)); - } - else { - u = -a; - } - sign = MPD_NEG; - } - else { - u = a; - } - _ssettriple(result, sign, u, 0); - mpd_qfinalize(result, ctx, status); -} - -/* quietly set a static decimal from an mpd_uint_t */ -void -mpd_qsset_uint(mpd_t *result, mpd_uint_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - _ssettriple(result, MPD_POS, a, 0); - mpd_qfinalize(result, ctx, status); -} - -/* quietly set a static decimal from an int32_t */ -void -mpd_qsset_i32(mpd_t *result, int32_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_qsset_ssize(result, a, ctx, status); -} - -/* quietly set a static decimal from a uint32_t */ -void -mpd_qsset_u32(mpd_t *result, uint32_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_qsset_uint(result, a, ctx, status); -} - -#ifdef CONFIG_64 -/* quietly set a static decimal from an int64_t */ -void -mpd_qsset_i64(mpd_t *result, int64_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_qsset_ssize(result, a, ctx, status); -} - -/* quietly set a static decimal from a uint64_t */ -void -mpd_qsset_u64(mpd_t *result, uint64_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_qsset_uint(result, a, ctx, status); -} -#endif - -/* quietly set a decimal from an mpd_ssize_t */ -void -mpd_qset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_minalloc(result); - mpd_qsset_ssize(result, a, ctx, status); -} - -/* quietly set a decimal from an mpd_uint_t */ -void -mpd_qset_uint(mpd_t *result, mpd_uint_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - _settriple(result, MPD_POS, a, 0); - mpd_qfinalize(result, ctx, status); -} - -/* quietly set a decimal from an int32_t */ -void -mpd_qset_i32(mpd_t *result, int32_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_qset_ssize(result, a, ctx, status); -} - -/* quietly set a decimal from a uint32_t */ -void -mpd_qset_u32(mpd_t *result, uint32_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_qset_uint(result, a, ctx, status); -} - -#if defined(CONFIG_32) && !defined(LEGACY_COMPILER) -/* set a decimal from a uint64_t */ -static void -_c32setu64(mpd_t *result, uint64_t u, uint8_t sign, uint32_t *status) -{ - mpd_uint_t w[3]; - uint64_t q; - int i, len; - - len = 0; - do { - q = u / MPD_RADIX; - w[len] = (mpd_uint_t)(u - q * MPD_RADIX); - u = q; len++; - } while (u != 0); - - if (!mpd_qresize(result, len, status)) { - return; - } - for (i = 0; i < len; i++) { - result->data[i] = w[i]; - } - - mpd_set_flags(result, sign); - result->exp = 0; - result->len = len; - mpd_setdigits(result); -} - -static void -_c32_qset_u64(mpd_t *result, uint64_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - _c32setu64(result, a, MPD_POS, status); - mpd_qfinalize(result, ctx, status); -} - -/* set a decimal from an int64_t */ -static void -_c32_qset_i64(mpd_t *result, int64_t a, const mpd_context_t *ctx, - uint32_t *status) -{ - uint64_t u; - uint8_t sign = MPD_POS; - - if (a < 0) { - if (a == INT64_MIN) { - u = (uint64_t)INT64_MAX + (-(INT64_MIN+INT64_MAX)); - } - else { - u = -a; - } - sign = MPD_NEG; - } - else { - u = a; - } - _c32setu64(result, u, sign, status); - mpd_qfinalize(result, ctx, status); -} -#endif /* CONFIG_32 && !LEGACY_COMPILER */ - -#ifndef LEGACY_COMPILER -/* quietly set a decimal from an int64_t */ -void -mpd_qset_i64(mpd_t *result, int64_t a, const mpd_context_t *ctx, - uint32_t *status) -{ -#ifdef CONFIG_64 - mpd_qset_ssize(result, a, ctx, status); -#else - _c32_qset_i64(result, a, ctx, status); -#endif -} - -/* quietly set a decimal from an int64_t, use a maxcontext for conversion */ -void -mpd_qset_i64_exact(mpd_t *result, int64_t a, uint32_t *status) -{ - mpd_context_t maxcontext; - - mpd_maxcontext(&maxcontext); -#ifdef CONFIG_64 - mpd_qset_ssize(result, a, &maxcontext, status); -#else - _c32_qset_i64(result, a, &maxcontext, status); -#endif - - if (*status & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { - /* we want exact results */ - mpd_seterror(result, MPD_Invalid_operation, status); - } - *status &= MPD_Errors; -} - -/* quietly set a decimal from a uint64_t */ -void -mpd_qset_u64(mpd_t *result, uint64_t a, const mpd_context_t *ctx, - uint32_t *status) -{ -#ifdef CONFIG_64 - mpd_qset_uint(result, a, ctx, status); -#else - _c32_qset_u64(result, a, ctx, status); -#endif -} - -/* quietly set a decimal from a uint64_t, use a maxcontext for conversion */ -void -mpd_qset_u64_exact(mpd_t *result, uint64_t a, uint32_t *status) -{ - mpd_context_t maxcontext; - - mpd_maxcontext(&maxcontext); -#ifdef CONFIG_64 - mpd_qset_uint(result, a, &maxcontext, status); -#else - _c32_qset_u64(result, a, &maxcontext, status); -#endif - - if (*status & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { - /* we want exact results */ - mpd_seterror(result, MPD_Invalid_operation, status); - } - *status &= MPD_Errors; -} -#endif /* !LEGACY_COMPILER */ - -/* - * Quietly get an mpd_uint_t from a decimal. Assumes - * MPD_UINT_DIGITS == MPD_RDIGITS+1, which is true for - * 32 and 64 bit machines. - * - * If the operation is impossible, MPD_Invalid_operation is set. - */ -static mpd_uint_t -_mpd_qget_uint(int use_sign, const mpd_t *a, uint32_t *status) -{ - mpd_t tmp; - mpd_uint_t tmp_data[2]; - mpd_uint_t lo, hi; - - if (mpd_isspecial(a)) { - *status |= MPD_Invalid_operation; - return MPD_UINT_MAX; - } - if (mpd_iszero(a)) { - return 0; - } - if (use_sign && mpd_isnegative(a)) { - *status |= MPD_Invalid_operation; - return MPD_UINT_MAX; - } - - if (a->digits+a->exp > MPD_RDIGITS+1) { - *status |= MPD_Invalid_operation; - return MPD_UINT_MAX; - } - - if (a->exp < 0) { - if (!_mpd_isint(a)) { - *status |= MPD_Invalid_operation; - return MPD_UINT_MAX; - } - /* At this point a->digits+a->exp <= MPD_RDIGITS+1, - * so the shift fits. */ - tmp.data = tmp_data; - tmp.flags = MPD_STATIC|MPD_STATIC_DATA; - tmp.alloc = 2; - mpd_qsshiftr(&tmp, a, -a->exp); - tmp.exp = 0; - a = &tmp; - } - - _mpd_get_msdigits(&hi, &lo, a, MPD_RDIGITS+1); - if (hi) { - *status |= MPD_Invalid_operation; - return MPD_UINT_MAX; - } - - if (a->exp > 0) { - _mpd_mul_words(&hi, &lo, lo, mpd_pow10[a->exp]); - if (hi) { - *status |= MPD_Invalid_operation; - return MPD_UINT_MAX; - } - } - - return lo; -} - -/* - * Sets Invalid_operation for: - * - specials - * - negative numbers (except negative zero) - * - non-integers - * - overflow - */ -mpd_uint_t -mpd_qget_uint(const mpd_t *a, uint32_t *status) -{ - return _mpd_qget_uint(1, a, status); -} - -/* Same as above, but gets the absolute value, i.e. the sign is ignored. */ -mpd_uint_t -mpd_qabs_uint(const mpd_t *a, uint32_t *status) -{ - return _mpd_qget_uint(0, a, status); -} - -/* quietly get an mpd_ssize_t from a decimal */ -mpd_ssize_t -mpd_qget_ssize(const mpd_t *a, uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_uint_t u; - int isneg; - - u = mpd_qabs_uint(a, &workstatus); - if (workstatus&MPD_Invalid_operation) { - *status |= workstatus; - return MPD_SSIZE_MAX; - } - - isneg = mpd_isnegative(a); - if (u <= MPD_SSIZE_MAX) { - return isneg ? -((mpd_ssize_t)u) : (mpd_ssize_t)u; - } - else if (isneg && u+(MPD_SSIZE_MIN+MPD_SSIZE_MAX) == MPD_SSIZE_MAX) { - return MPD_SSIZE_MIN; - } - - *status |= MPD_Invalid_operation; - return MPD_SSIZE_MAX; -} - -#if defined(CONFIG_32) && !defined(LEGACY_COMPILER) -/* - * Quietly get a uint64_t from a decimal. If the operation is impossible, - * MPD_Invalid_operation is set. - */ -static uint64_t -_c32_qget_u64(int use_sign, const mpd_t *a, uint32_t *status) -{ - MPD_NEW_STATIC(tmp,0,0,20,3); - mpd_context_t maxcontext; - uint64_t ret; - - tmp_data[0] = 709551615; - tmp_data[1] = 446744073; - tmp_data[2] = 18; - - if (mpd_isspecial(a)) { - *status |= MPD_Invalid_operation; - return UINT64_MAX; - } - if (mpd_iszero(a)) { - return 0; - } - if (use_sign && mpd_isnegative(a)) { - *status |= MPD_Invalid_operation; - return UINT64_MAX; - } - if (!_mpd_isint(a)) { - *status |= MPD_Invalid_operation; - return UINT64_MAX; - } - - if (_mpd_cmp_abs(a, &tmp) > 0) { - *status |= MPD_Invalid_operation; - return UINT64_MAX; - } - - mpd_maxcontext(&maxcontext); - mpd_qrescale(&tmp, a, 0, &maxcontext, &maxcontext.status); - maxcontext.status &= ~MPD_Rounded; - if (maxcontext.status != 0) { - *status |= (maxcontext.status|MPD_Invalid_operation); /* GCOV_NOT_REACHED */ - return UINT64_MAX; /* GCOV_NOT_REACHED */ - } - - ret = 0; - switch (tmp.len) { - case 3: - ret += (uint64_t)tmp_data[2] * 1000000000000000000ULL; - case 2: - ret += (uint64_t)tmp_data[1] * 1000000000ULL; - case 1: - ret += tmp_data[0]; - break; - default: - abort(); /* GCOV_NOT_REACHED */ - } - - return ret; -} - -static int64_t -_c32_qget_i64(const mpd_t *a, uint32_t *status) -{ - uint64_t u; - int isneg; - - u = _c32_qget_u64(0, a, status); - if (*status&MPD_Invalid_operation) { - return INT64_MAX; - } - - isneg = mpd_isnegative(a); - if (u <= INT64_MAX) { - return isneg ? -((int64_t)u) : (int64_t)u; - } - else if (isneg && u+(INT64_MIN+INT64_MAX) == INT64_MAX) { - return INT64_MIN; - } - - *status |= MPD_Invalid_operation; - return INT64_MAX; -} -#endif /* CONFIG_32 && !LEGACY_COMPILER */ - -#ifdef CONFIG_64 -/* quietly get a uint64_t from a decimal */ -uint64_t -mpd_qget_u64(const mpd_t *a, uint32_t *status) -{ - return mpd_qget_uint(a, status); -} - -/* quietly get an int64_t from a decimal */ -int64_t -mpd_qget_i64(const mpd_t *a, uint32_t *status) -{ - return mpd_qget_ssize(a, status); -} - -/* quietly get a uint32_t from a decimal */ -uint32_t -mpd_qget_u32(const mpd_t *a, uint32_t *status) -{ - uint32_t workstatus = 0; - uint64_t x = mpd_qget_uint(a, &workstatus); - - if (workstatus&MPD_Invalid_operation) { - *status |= workstatus; - return UINT32_MAX; - } - if (x > UINT32_MAX) { - *status |= MPD_Invalid_operation; - return UINT32_MAX; - } - - return (uint32_t)x; -} - -/* quietly get an int32_t from a decimal */ -int32_t -mpd_qget_i32(const mpd_t *a, uint32_t *status) -{ - uint32_t workstatus = 0; - int64_t x = mpd_qget_ssize(a, &workstatus); - - if (workstatus&MPD_Invalid_operation) { - *status |= workstatus; - return INT32_MAX; - } - if (x < INT32_MIN || x > INT32_MAX) { - *status |= MPD_Invalid_operation; - return INT32_MAX; - } - - return (int32_t)x; -} -#else -#ifndef LEGACY_COMPILER -/* quietly get a uint64_t from a decimal */ -uint64_t -mpd_qget_u64(const mpd_t *a, uint32_t *status) -{ - uint32_t workstatus = 0; - uint64_t x = _c32_qget_u64(1, a, &workstatus); - *status |= workstatus; - return x; -} - -/* quietly get an int64_t from a decimal */ -int64_t -mpd_qget_i64(const mpd_t *a, uint32_t *status) -{ - uint32_t workstatus = 0; - int64_t x = _c32_qget_i64(a, &workstatus); - *status |= workstatus; - return x; -} -#endif - -/* quietly get a uint32_t from a decimal */ -uint32_t -mpd_qget_u32(const mpd_t *a, uint32_t *status) -{ - return mpd_qget_uint(a, status); -} - -/* quietly get an int32_t from a decimal */ -int32_t -mpd_qget_i32(const mpd_t *a, uint32_t *status) -{ - return mpd_qget_ssize(a, status); -} -#endif - - -/******************************************************************************/ -/* Filtering input of functions, finalizing output of functions */ -/******************************************************************************/ - -/* - * Check if the operand is NaN, copy to result and return 1 if this is - * the case. Copying can fail since NaNs are allowed to have a payload that - * does not fit in MPD_MINALLOC. - */ -int -mpd_qcheck_nan(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - if (mpd_isnan(a)) { - *status |= mpd_issnan(a) ? MPD_Invalid_operation : 0; - mpd_qcopy(result, a, status); - mpd_set_qnan(result); - _mpd_fix_nan(result, ctx); - return 1; - } - return 0; -} - -/* - * Check if either operand is NaN, copy to result and return 1 if this - * is the case. Copying can fail since NaNs are allowed to have a payload - * that does not fit in MPD_MINALLOC. - */ -int -mpd_qcheck_nans(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - if ((a->flags|b->flags)&(MPD_NAN|MPD_SNAN)) { - const mpd_t *choice = b; - if (mpd_issnan(a)) { - choice = a; - *status |= MPD_Invalid_operation; - } - else if (mpd_issnan(b)) { - *status |= MPD_Invalid_operation; - } - else if (mpd_isqnan(a)) { - choice = a; - } - mpd_qcopy(result, choice, status); - mpd_set_qnan(result); - _mpd_fix_nan(result, ctx); - return 1; - } - return 0; -} - -/* - * Check if one of the operands is NaN, copy to result and return 1 if this - * is the case. Copying can fail since NaNs are allowed to have a payload - * that does not fit in MPD_MINALLOC. - */ -static int -mpd_qcheck_3nans(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, - const mpd_context_t *ctx, uint32_t *status) -{ - if ((a->flags|b->flags|c->flags)&(MPD_NAN|MPD_SNAN)) { - const mpd_t *choice = c; - if (mpd_issnan(a)) { - choice = a; - *status |= MPD_Invalid_operation; - } - else if (mpd_issnan(b)) { - choice = b; - *status |= MPD_Invalid_operation; - } - else if (mpd_issnan(c)) { - *status |= MPD_Invalid_operation; - } - else if (mpd_isqnan(a)) { - choice = a; - } - else if (mpd_isqnan(b)) { - choice = b; - } - mpd_qcopy(result, choice, status); - mpd_set_qnan(result); - _mpd_fix_nan(result, ctx); - return 1; - } - return 0; -} - -/* Check if rounding digit 'rnd' leads to an increment. */ -static inline int -_mpd_rnd_incr(const mpd_t *dec, mpd_uint_t rnd, const mpd_context_t *ctx) -{ - int ld; - - switch (ctx->round) { - case MPD_ROUND_DOWN: case MPD_ROUND_TRUNC: - return 0; - case MPD_ROUND_HALF_UP: - return (rnd >= 5); - case MPD_ROUND_HALF_EVEN: - return (rnd > 5) || ((rnd == 5) && mpd_isoddcoeff(dec)); - case MPD_ROUND_CEILING: - return !(rnd == 0 || mpd_isnegative(dec)); - case MPD_ROUND_FLOOR: - return !(rnd == 0 || mpd_ispositive(dec)); - case MPD_ROUND_HALF_DOWN: - return (rnd > 5); - case MPD_ROUND_UP: - return !(rnd == 0); - case MPD_ROUND_05UP: - ld = (int)mpd_lsd(dec->data[0]); - return (!(rnd == 0) && (ld == 0 || ld == 5)); - default: - /* Without a valid context, further results will be undefined. */ - return 0; /* GCOV_NOT_REACHED */ - } -} - -/* - * Apply rounding to a decimal that has been right-shifted into a full - * precision decimal. If an increment leads to an overflow of the precision, - * adjust the coefficient and the exponent and check the new exponent for - * overflow. - */ -static inline void -_mpd_apply_round(mpd_t *dec, mpd_uint_t rnd, const mpd_context_t *ctx, - uint32_t *status) -{ - if (_mpd_rnd_incr(dec, rnd, ctx)) { - /* We have a number with exactly ctx->prec digits. The increment - * can only lead to an overflow if the decimal is all nines. In - * that case, the result is a power of ten with prec+1 digits. - * - * If the precision is a multiple of MPD_RDIGITS, this situation is - * detected by _mpd_baseincr returning a carry. - * If the precision is not a multiple of MPD_RDIGITS, we have to - * check if the result has one digit too many. - */ - mpd_uint_t carry = _mpd_baseincr(dec->data, dec->len); - if (carry) { - dec->data[dec->len-1] = mpd_pow10[MPD_RDIGITS-1]; - dec->exp += 1; - _mpd_check_exp(dec, ctx, status); - return; - } - mpd_setdigits(dec); - if (dec->digits > ctx->prec) { - mpd_qshiftr_inplace(dec, 1); - dec->exp += 1; - dec->digits = ctx->prec; - _mpd_check_exp(dec, ctx, status); - } - } -} - -/* - * Apply rounding to a decimal. Allow overflow of the precision. - */ -static inline void -_mpd_apply_round_excess(mpd_t *dec, mpd_uint_t rnd, const mpd_context_t *ctx, - uint32_t *status) -{ - if (_mpd_rnd_incr(dec, rnd, ctx)) { - mpd_uint_t carry = _mpd_baseincr(dec->data, dec->len); - if (carry) { - if (!mpd_qresize(dec, dec->len+1, status)) { - return; - } - dec->data[dec->len] = 1; - dec->len += 1; - } - mpd_setdigits(dec); - } -} - -/* - * Apply rounding to a decimal that has been right-shifted into a decimal - * with full precision or less. Return failure if an increment would - * overflow the precision. - */ -static inline int -_mpd_apply_round_fit(mpd_t *dec, mpd_uint_t rnd, const mpd_context_t *ctx, - uint32_t *status) -{ - if (_mpd_rnd_incr(dec, rnd, ctx)) { - mpd_uint_t carry = _mpd_baseincr(dec->data, dec->len); - if (carry) { - if (!mpd_qresize(dec, dec->len+1, status)) { - return 0; - } - dec->data[dec->len] = 1; - dec->len += 1; - } - mpd_setdigits(dec); - if (dec->digits > ctx->prec) { - mpd_seterror(dec, MPD_Invalid_operation, status); - return 0; - } - } - return 1; -} - -/* Check a normal number for overflow, underflow, clamping. If the operand - is modified, it will be zero, special or (sub)normal with a coefficient - that fits into the current context precision. */ -static inline void -_mpd_check_exp(mpd_t *dec, const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t adjexp, etiny, shift; - int rnd; - - adjexp = mpd_adjexp(dec); - if (adjexp > ctx->emax) { - - if (mpd_iszerocoeff(dec)) { - dec->exp = ctx->emax; - if (ctx->clamp) { - dec->exp -= (ctx->prec-1); - } - mpd_zerocoeff(dec); - *status |= MPD_Clamped; - return; - } - - switch (ctx->round) { - case MPD_ROUND_HALF_UP: case MPD_ROUND_HALF_EVEN: - case MPD_ROUND_HALF_DOWN: case MPD_ROUND_UP: - case MPD_ROUND_TRUNC: - mpd_setspecial(dec, mpd_sign(dec), MPD_INF); - break; - case MPD_ROUND_DOWN: case MPD_ROUND_05UP: - mpd_qmaxcoeff(dec, ctx, status); - dec->exp = ctx->emax - ctx->prec + 1; - break; - case MPD_ROUND_CEILING: - if (mpd_isnegative(dec)) { - mpd_qmaxcoeff(dec, ctx, status); - dec->exp = ctx->emax - ctx->prec + 1; - } - else { - mpd_setspecial(dec, MPD_POS, MPD_INF); - } - break; - case MPD_ROUND_FLOOR: - if (mpd_ispositive(dec)) { - mpd_qmaxcoeff(dec, ctx, status); - dec->exp = ctx->emax - ctx->prec + 1; - } - else { - mpd_setspecial(dec, MPD_NEG, MPD_INF); - } - break; - default: /* debug */ - abort(); /* GCOV_NOT_REACHED */ - } - - *status |= MPD_Overflow|MPD_Inexact|MPD_Rounded; - - } /* fold down */ - else if (ctx->clamp && dec->exp > mpd_etop(ctx)) { - /* At this point adjexp=exp+digits-1 <= emax and exp > etop=emax-prec+1: - * (1) shift = exp -emax+prec-1 > 0 - * (2) digits+shift = exp+digits-1 - emax + prec <= prec */ - shift = dec->exp - mpd_etop(ctx); - if (!mpd_qshiftl(dec, dec, shift, status)) { - return; - } - dec->exp -= shift; - *status |= MPD_Clamped; - if (!mpd_iszerocoeff(dec) && adjexp < ctx->emin) { - /* Underflow is impossible, since exp < etiny=emin-prec+1 - * and exp > etop=emax-prec+1 would imply emax < emin. */ - *status |= MPD_Subnormal; - } - } - else if (adjexp < ctx->emin) { - - etiny = mpd_etiny(ctx); - - if (mpd_iszerocoeff(dec)) { - if (dec->exp < etiny) { - dec->exp = etiny; - mpd_zerocoeff(dec); - *status |= MPD_Clamped; - } - return; - } - - *status |= MPD_Subnormal; - if (dec->exp < etiny) { - /* At this point adjexp=exp+digits-1 < emin and exp < etiny=emin-prec+1: - * (1) shift = emin-prec+1 - exp > 0 - * (2) digits-shift = exp+digits-1 - emin + prec < prec */ - shift = etiny - dec->exp; - rnd = (int)mpd_qshiftr_inplace(dec, shift); - dec->exp = etiny; - /* We always have a spare digit in case of an increment. */ - _mpd_apply_round_excess(dec, rnd, ctx, status); - *status |= MPD_Rounded; - if (rnd) { - *status |= (MPD_Inexact|MPD_Underflow); - if (mpd_iszerocoeff(dec)) { - mpd_zerocoeff(dec); - *status |= MPD_Clamped; - } - } - } - /* Case exp >= etiny=emin-prec+1: - * (1) adjexp=exp+digits-1 < emin - * (2) digits < emin-exp+1 <= prec */ - } -} - -/* Transcendental functions do not always set Underflow reliably, - * since they only use as much precision as is necessary for correct - * rounding. If a result like 1.0000000000e-101 is finalized, there - * is no rounding digit that would trigger Underflow. But we can - * assume Inexact, so a short check suffices. */ -static inline void -mpd_check_underflow(mpd_t *dec, const mpd_context_t *ctx, uint32_t *status) -{ - if (mpd_adjexp(dec) < ctx->emin && !mpd_iszero(dec) && - dec->exp < mpd_etiny(ctx)) { - *status |= MPD_Underflow; - } -} - -/* Check if a normal number must be rounded after the exponent has been checked. */ -static inline void -_mpd_check_round(mpd_t *dec, const mpd_context_t *ctx, uint32_t *status) -{ - mpd_uint_t rnd; - mpd_ssize_t shift; - - /* must handle specials: _mpd_check_exp() can produce infinities or NaNs */ - if (mpd_isspecial(dec)) { - return; - } - - if (dec->digits > ctx->prec) { - shift = dec->digits - ctx->prec; - rnd = mpd_qshiftr_inplace(dec, shift); - dec->exp += shift; - _mpd_apply_round(dec, rnd, ctx, status); - *status |= MPD_Rounded; - if (rnd) { - *status |= MPD_Inexact; - } - } -} - -/* Finalize all operations. */ -void -mpd_qfinalize(mpd_t *result, const mpd_context_t *ctx, uint32_t *status) -{ - if (mpd_isspecial(result)) { - if (mpd_isnan(result)) { - _mpd_fix_nan(result, ctx); - } - return; - } - - _mpd_check_exp(result, ctx, status); - _mpd_check_round(result, ctx, status); -} - - -/******************************************************************************/ -/* Copying */ -/******************************************************************************/ - -/* Internal function: Copy a decimal, share data with src: USE WITH CARE! */ -static inline void -_mpd_copy_shared(mpd_t *dest, const mpd_t *src) -{ - dest->flags = src->flags; - dest->exp = src->exp; - dest->digits = src->digits; - dest->len = src->len; - dest->alloc = src->alloc; - dest->data = src->data; - - mpd_set_shared_data(dest); -} - -/* - * Copy a decimal. In case of an error, status is set to MPD_Malloc_error. - */ -int -mpd_qcopy(mpd_t *result, const mpd_t *a, uint32_t *status) -{ - if (result == a) return 1; - - if (!mpd_qresize(result, a->len, status)) { - return 0; - } - - mpd_copy_flags(result, a); - result->exp = a->exp; - result->digits = a->digits; - result->len = a->len; - memcpy(result->data, a->data, a->len * (sizeof *result->data)); - - return 1; -} - -/* Same as mpd_qcopy, but do not set the result to NaN on failure. */ -int -mpd_qcopy_cxx(mpd_t *result, const mpd_t *a) -{ - if (result == a) return 1; - - if (!mpd_qresize_cxx(result, a->len)) { - return 0; - } - - mpd_copy_flags(result, a); - result->exp = a->exp; - result->digits = a->digits; - result->len = a->len; - memcpy(result->data, a->data, a->len * (sizeof *result->data)); - - return 1; -} - -/* - * Copy to a decimal with a static buffer. The caller has to make sure that - * the buffer is big enough. Cannot fail. - */ -static void -mpd_qcopy_static(mpd_t *result, const mpd_t *a) -{ - if (result == a) return; - - memcpy(result->data, a->data, a->len * (sizeof *result->data)); - - mpd_copy_flags(result, a); - result->exp = a->exp; - result->digits = a->digits; - result->len = a->len; -} - -/* - * Return a newly allocated copy of the operand. In case of an error, - * status is set to MPD_Malloc_error and the return value is NULL. - */ -mpd_t * -mpd_qncopy(const mpd_t *a) -{ - mpd_t *result; - - if ((result = mpd_qnew_size(a->len)) == NULL) { - return NULL; - } - memcpy(result->data, a->data, a->len * (sizeof *result->data)); - mpd_copy_flags(result, a); - result->exp = a->exp; - result->digits = a->digits; - result->len = a->len; - - return result; -} - -/* - * Copy a decimal and set the sign to positive. In case of an error, the - * status is set to MPD_Malloc_error. - */ -int -mpd_qcopy_abs(mpd_t *result, const mpd_t *a, uint32_t *status) -{ - if (!mpd_qcopy(result, a, status)) { - return 0; - } - mpd_set_positive(result); - return 1; -} - -/* - * Copy a decimal and negate the sign. In case of an error, the - * status is set to MPD_Malloc_error. - */ -int -mpd_qcopy_negate(mpd_t *result, const mpd_t *a, uint32_t *status) -{ - if (!mpd_qcopy(result, a, status)) { - return 0; - } - _mpd_negate(result); - return 1; -} - -/* - * Copy a decimal, setting the sign of the first operand to the sign of the - * second operand. In case of an error, the status is set to MPD_Malloc_error. - */ -int -mpd_qcopy_sign(mpd_t *result, const mpd_t *a, const mpd_t *b, uint32_t *status) -{ - uint8_t sign_b = mpd_sign(b); /* result may equal b! */ - - if (!mpd_qcopy(result, a, status)) { - return 0; - } - mpd_set_sign(result, sign_b); - return 1; -} - - -/******************************************************************************/ -/* Comparisons */ -/******************************************************************************/ - -/* - * For all functions that compare two operands and return an int the usual - * convention applies to the return value: - * - * -1 if op1 < op2 - * 0 if op1 == op2 - * 1 if op1 > op2 - * - * INT_MAX for error - */ - - -/* Convenience macro. If a and b are not equal, return from the calling - * function with the correct comparison value. */ -#define CMP_EQUAL_OR_RETURN(a, b) \ - if (a != b) { \ - if (a < b) { \ - return -1; \ - } \ - return 1; \ - } - -/* - * Compare the data of big and small. This function does the equivalent - * of first shifting small to the left and then comparing the data of - * big and small, except that no allocation for the left shift is needed. - */ -static int -_mpd_basecmp(mpd_uint_t *big, mpd_uint_t *small, mpd_size_t n, mpd_size_t m, - mpd_size_t shift) -{ -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) && !defined(__clang__) - /* spurious uninitialized warnings */ - mpd_uint_t l=l, lprev=lprev, h=h; -#else - mpd_uint_t l, lprev, h; -#endif - mpd_uint_t q, r; - mpd_uint_t ph, x; - - assert(m > 0 && n >= m && shift > 0); - - _mpd_div_word(&q, &r, (mpd_uint_t)shift, MPD_RDIGITS); - - if (r != 0) { - - ph = mpd_pow10[r]; - - --m; --n; - _mpd_divmod_pow10(&h, &lprev, small[m--], MPD_RDIGITS-r); - if (h != 0) { - CMP_EQUAL_OR_RETURN(big[n], h) - --n; - } - for (; m != MPD_SIZE_MAX; m--,n--) { - _mpd_divmod_pow10(&h, &l, small[m], MPD_RDIGITS-r); - x = ph * lprev + h; - CMP_EQUAL_OR_RETURN(big[n], x) - lprev = l; - } - x = ph * lprev; - CMP_EQUAL_OR_RETURN(big[q], x) - } - else { - while (--m != MPD_SIZE_MAX) { - CMP_EQUAL_OR_RETURN(big[m+q], small[m]) - } - } - - return !_mpd_isallzero(big, q); -} - -/* Compare two decimals with the same adjusted exponent. */ -static int -_mpd_cmp_same_adjexp(const mpd_t *a, const mpd_t *b) -{ - mpd_ssize_t shift, i; - - if (a->exp != b->exp) { - /* Cannot wrap: a->exp + a->digits = b->exp + b->digits, so - * a->exp - b->exp = b->digits - a->digits. */ - shift = a->exp - b->exp; - if (shift > 0) { - return -1 * _mpd_basecmp(b->data, a->data, b->len, a->len, shift); - } - else { - return _mpd_basecmp(a->data, b->data, a->len, b->len, -shift); - } - } - - /* - * At this point adjexp(a) == adjexp(b) and a->exp == b->exp, - * so a->digits == b->digits, therefore a->len == b->len. - */ - for (i = a->len-1; i >= 0; --i) { - CMP_EQUAL_OR_RETURN(a->data[i], b->data[i]) - } - - return 0; -} - -/* Compare two numerical values. */ -static int -_mpd_cmp(const mpd_t *a, const mpd_t *b) -{ - mpd_ssize_t adjexp_a, adjexp_b; - - /* equal pointers */ - if (a == b) { - return 0; - } - - /* infinities */ - if (mpd_isinfinite(a)) { - if (mpd_isinfinite(b)) { - return mpd_isnegative(b) - mpd_isnegative(a); - } - return mpd_arith_sign(a); - } - if (mpd_isinfinite(b)) { - return -mpd_arith_sign(b); - } - - /* zeros */ - if (mpd_iszerocoeff(a)) { - if (mpd_iszerocoeff(b)) { - return 0; - } - return -mpd_arith_sign(b); - } - if (mpd_iszerocoeff(b)) { - return mpd_arith_sign(a); - } - - /* different signs */ - if (mpd_sign(a) != mpd_sign(b)) { - return mpd_sign(b) - mpd_sign(a); - } - - /* different adjusted exponents */ - adjexp_a = mpd_adjexp(a); - adjexp_b = mpd_adjexp(b); - if (adjexp_a != adjexp_b) { - if (adjexp_a < adjexp_b) { - return -1 * mpd_arith_sign(a); - } - return mpd_arith_sign(a); - } - - /* same adjusted exponents */ - return _mpd_cmp_same_adjexp(a, b) * mpd_arith_sign(a); -} - -/* Compare the absolutes of two numerical values. */ -static int -_mpd_cmp_abs(const mpd_t *a, const mpd_t *b) -{ - mpd_ssize_t adjexp_a, adjexp_b; - - /* equal pointers */ - if (a == b) { - return 0; - } - - /* infinities */ - if (mpd_isinfinite(a)) { - if (mpd_isinfinite(b)) { - return 0; - } - return 1; - } - if (mpd_isinfinite(b)) { - return -1; - } - - /* zeros */ - if (mpd_iszerocoeff(a)) { - if (mpd_iszerocoeff(b)) { - return 0; - } - return -1; - } - if (mpd_iszerocoeff(b)) { - return 1; - } - - /* different adjusted exponents */ - adjexp_a = mpd_adjexp(a); - adjexp_b = mpd_adjexp(b); - if (adjexp_a != adjexp_b) { - if (adjexp_a < adjexp_b) { - return -1; - } - return 1; - } - - /* same adjusted exponents */ - return _mpd_cmp_same_adjexp(a, b); -} - -/* Compare two values and return an integer result. */ -int -mpd_qcmp(const mpd_t *a, const mpd_t *b, uint32_t *status) -{ - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_isnan(a) || mpd_isnan(b)) { - *status |= MPD_Invalid_operation; - return INT_MAX; - } - } - - return _mpd_cmp(a, b); -} - -/* - * Compare a and b, convert the usual integer result to a decimal and - * store it in 'result'. For convenience, the integer result of the comparison - * is returned. Comparisons involving NaNs return NaN/INT_MAX. - */ -int -mpd_qcompare(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - int c; - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return INT_MAX; - } - } - - c = _mpd_cmp(a, b); - _settriple(result, (c < 0), (c != 0), 0); - return c; -} - -/* Same as mpd_compare(), but signal for all NaNs, i.e. also for quiet NaNs. */ -int -mpd_qcompare_signal(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - int c; - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - *status |= MPD_Invalid_operation; - return INT_MAX; - } - } - - c = _mpd_cmp(a, b); - _settriple(result, (c < 0), (c != 0), 0); - return c; -} - -/* Compare the operands using a total order. */ -int -mpd_cmp_total(const mpd_t *a, const mpd_t *b) -{ - mpd_t aa, bb; - int nan_a, nan_b; - int c; - - if (mpd_sign(a) != mpd_sign(b)) { - return mpd_sign(b) - mpd_sign(a); - } - - - if (mpd_isnan(a)) { - c = 1; - if (mpd_isnan(b)) { - nan_a = (mpd_isqnan(a)) ? 1 : 0; - nan_b = (mpd_isqnan(b)) ? 1 : 0; - if (nan_b == nan_a) { - if (a->len > 0 && b->len > 0) { - _mpd_copy_shared(&aa, a); - _mpd_copy_shared(&bb, b); - aa.exp = bb.exp = 0; - /* compare payload */ - c = _mpd_cmp_abs(&aa, &bb); - } - else { - c = (a->len > 0) - (b->len > 0); - } - } - else { - c = nan_a - nan_b; - } - } - } - else if (mpd_isnan(b)) { - c = -1; - } - else { - c = _mpd_cmp_abs(a, b); - if (c == 0 && a->exp != b->exp) { - c = (a->exp < b->exp) ? -1 : 1; - } - } - - return c * mpd_arith_sign(a); -} - -/* - * Compare a and b according to a total order, convert the usual integer result - * to a decimal and store it in 'result'. For convenience, the integer result - * of the comparison is returned. - */ -int -mpd_compare_total(mpd_t *result, const mpd_t *a, const mpd_t *b) -{ - int c; - - c = mpd_cmp_total(a, b); - _settriple(result, (c < 0), (c != 0), 0); - return c; -} - -/* Compare the magnitude of the operands using a total order. */ -int -mpd_cmp_total_mag(const mpd_t *a, const mpd_t *b) -{ - mpd_t aa, bb; - - _mpd_copy_shared(&aa, a); - _mpd_copy_shared(&bb, b); - - mpd_set_positive(&aa); - mpd_set_positive(&bb); - - return mpd_cmp_total(&aa, &bb); -} - -/* - * Compare the magnitude of a and b according to a total order, convert the - * the usual integer result to a decimal and store it in 'result'. - * For convenience, the integer result of the comparison is returned. - */ -int -mpd_compare_total_mag(mpd_t *result, const mpd_t *a, const mpd_t *b) -{ - int c; - - c = mpd_cmp_total_mag(a, b); - _settriple(result, (c < 0), (c != 0), 0); - return c; -} - -/* Determine an ordering for operands that are numerically equal. */ -static inline int -_mpd_cmp_numequal(const mpd_t *a, const mpd_t *b) -{ - int sign_a, sign_b; - int c; - - sign_a = mpd_sign(a); - sign_b = mpd_sign(b); - if (sign_a != sign_b) { - c = sign_b - sign_a; - } - else { - c = (a->exp < b->exp) ? -1 : 1; - c *= mpd_arith_sign(a); - } - - return c; -} - - -/******************************************************************************/ -/* Shifting the coefficient */ -/******************************************************************************/ - -/* - * Shift the coefficient of the operand to the left, no check for specials. - * Both operands may be the same pointer. If the result length has to be - * increased, mpd_qresize() might fail with MPD_Malloc_error. - */ -int -mpd_qshiftl(mpd_t *result, const mpd_t *a, mpd_ssize_t n, uint32_t *status) -{ - mpd_ssize_t size; - - assert(!mpd_isspecial(a)); - assert(n >= 0); - - if (mpd_iszerocoeff(a) || n == 0) { - return mpd_qcopy(result, a, status); - } - - size = mpd_digits_to_size(a->digits+n); - if (!mpd_qresize(result, size, status)) { - return 0; /* result is NaN */ - } - - _mpd_baseshiftl(result->data, a->data, size, a->len, n); - - mpd_copy_flags(result, a); - result->exp = a->exp; - result->digits = a->digits+n; - result->len = size; - - return 1; -} - -/* Determine the rounding indicator if all digits of the coefficient are shifted - * out of the picture. */ -static mpd_uint_t -_mpd_get_rnd(const mpd_uint_t *data, mpd_ssize_t len, int use_msd) -{ - mpd_uint_t rnd = 0, rest = 0, word; - - word = data[len-1]; - /* special treatment for the most significant digit if shift == digits */ - if (use_msd) { - _mpd_divmod_pow10(&rnd, &rest, word, mpd_word_digits(word)-1); - if (len > 1 && rest == 0) { - rest = !_mpd_isallzero(data, len-1); - } - } - else { - rest = !_mpd_isallzero(data, len); - } - - return (rnd == 0 || rnd == 5) ? rnd + !!rest : rnd; -} - -/* - * Same as mpd_qshiftr(), but 'result' is an mpd_t with a static coefficient. - * It is the caller's responsibility to ensure that the coefficient is big - * enough. The function cannot fail. - */ -static mpd_uint_t -mpd_qsshiftr(mpd_t *result, const mpd_t *a, mpd_ssize_t n) -{ - mpd_uint_t rnd; - mpd_ssize_t size; - - assert(!mpd_isspecial(a)); - assert(n >= 0); - - if (mpd_iszerocoeff(a) || n == 0) { - mpd_qcopy_static(result, a); - return 0; - } - - if (n >= a->digits) { - rnd = _mpd_get_rnd(a->data, a->len, (n==a->digits)); - mpd_zerocoeff(result); - } - else { - result->digits = a->digits-n; - size = mpd_digits_to_size(result->digits); - rnd = _mpd_baseshiftr(result->data, a->data, a->len, n); - result->len = size; - } - - mpd_copy_flags(result, a); - result->exp = a->exp; - - return rnd; -} - -/* - * Inplace shift of the coefficient to the right, no check for specials. - * Returns the rounding indicator for mpd_rnd_incr(). - * The function cannot fail. - */ -mpd_uint_t -mpd_qshiftr_inplace(mpd_t *result, mpd_ssize_t n) -{ - uint32_t dummy; - mpd_uint_t rnd; - mpd_ssize_t size; - - assert(!mpd_isspecial(result)); - assert(n >= 0); - - if (mpd_iszerocoeff(result) || n == 0) { - return 0; - } - - if (n >= result->digits) { - rnd = _mpd_get_rnd(result->data, result->len, (n==result->digits)); - mpd_zerocoeff(result); - } - else { - rnd = _mpd_baseshiftr(result->data, result->data, result->len, n); - result->digits -= n; - size = mpd_digits_to_size(result->digits); - /* reducing the size cannot fail */ - mpd_qresize(result, size, &dummy); - result->len = size; - } - - return rnd; -} - -/* - * Shift the coefficient of the operand to the right, no check for specials. - * Both operands may be the same pointer. Returns the rounding indicator to - * be used by mpd_rnd_incr(). If the result length has to be increased, - * mpd_qcopy() or mpd_qresize() might fail with MPD_Malloc_error. In those - * cases, MPD_UINT_MAX is returned. - */ -mpd_uint_t -mpd_qshiftr(mpd_t *result, const mpd_t *a, mpd_ssize_t n, uint32_t *status) -{ - mpd_uint_t rnd; - mpd_ssize_t size; - - assert(!mpd_isspecial(a)); - assert(n >= 0); - - if (mpd_iszerocoeff(a) || n == 0) { - if (!mpd_qcopy(result, a, status)) { - return MPD_UINT_MAX; - } - return 0; - } - - if (n >= a->digits) { - rnd = _mpd_get_rnd(a->data, a->len, (n==a->digits)); - mpd_zerocoeff(result); - } - else { - result->digits = a->digits-n; - size = mpd_digits_to_size(result->digits); - if (result == a) { - rnd = _mpd_baseshiftr(result->data, a->data, a->len, n); - /* reducing the size cannot fail */ - mpd_qresize(result, size, status); - } - else { - if (!mpd_qresize(result, size, status)) { - return MPD_UINT_MAX; - } - rnd = _mpd_baseshiftr(result->data, a->data, a->len, n); - } - result->len = size; - } - - mpd_copy_flags(result, a); - result->exp = a->exp; - - return rnd; -} - - -/******************************************************************************/ -/* Miscellaneous operations */ -/******************************************************************************/ - -/* Logical And */ -void -mpd_qand(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - const mpd_t *big = a, *small = b; - mpd_uint_t x, y, z, xbit, ybit; - int k, mswdigits; - mpd_ssize_t i; - - if (mpd_isspecial(a) || mpd_isspecial(b) || - mpd_isnegative(a) || mpd_isnegative(b) || - a->exp != 0 || b->exp != 0) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (b->digits > a->digits) { - big = b; - small = a; - } - if (!mpd_qresize(result, big->len, status)) { - return; - } - - - /* full words */ - for (i = 0; i < small->len-1; i++) { - x = small->data[i]; - y = big->data[i]; - z = 0; - for (k = 0; k < MPD_RDIGITS; k++) { - xbit = x % 10; - x /= 10; - ybit = y % 10; - y /= 10; - if (xbit > 1 || ybit > 1) { - goto invalid_operation; - } - z += (xbit&ybit) ? mpd_pow10[k] : 0; - } - result->data[i] = z; - } - /* most significant word of small */ - x = small->data[i]; - y = big->data[i]; - z = 0; - mswdigits = mpd_word_digits(x); - for (k = 0; k < mswdigits; k++) { - xbit = x % 10; - x /= 10; - ybit = y % 10; - y /= 10; - if (xbit > 1 || ybit > 1) { - goto invalid_operation; - } - z += (xbit&ybit) ? mpd_pow10[k] : 0; - } - result->data[i++] = z; - - /* scan the rest of y for digits > 1 */ - for (; k < MPD_RDIGITS; k++) { - ybit = y % 10; - y /= 10; - if (ybit > 1) { - goto invalid_operation; - } - } - /* scan the rest of big for digits > 1 */ - for (; i < big->len; i++) { - y = big->data[i]; - for (k = 0; k < MPD_RDIGITS; k++) { - ybit = y % 10; - y /= 10; - if (ybit > 1) { - goto invalid_operation; - } - } - } - - mpd_clear_flags(result); - result->exp = 0; - result->len = _mpd_real_size(result->data, small->len); - mpd_qresize(result, result->len, status); - mpd_setdigits(result); - _mpd_cap(result, ctx); - return; - -invalid_operation: - mpd_seterror(result, MPD_Invalid_operation, status); -} - -/* Class of an operand. Returns a pointer to the constant name. */ -const char * -mpd_class(const mpd_t *a, const mpd_context_t *ctx) -{ - if (mpd_isnan(a)) { - if (mpd_isqnan(a)) - return "NaN"; - else - return "sNaN"; - } - else if (mpd_ispositive(a)) { - if (mpd_isinfinite(a)) - return "+Infinity"; - else if (mpd_iszero(a)) - return "+Zero"; - else if (mpd_isnormal(a, ctx)) - return "+Normal"; - else - return "+Subnormal"; - } - else { - if (mpd_isinfinite(a)) - return "-Infinity"; - else if (mpd_iszero(a)) - return "-Zero"; - else if (mpd_isnormal(a, ctx)) - return "-Normal"; - else - return "-Subnormal"; - } -} - -/* Logical Xor */ -void -mpd_qinvert(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_uint_t x, z, xbit; - mpd_ssize_t i, digits, len; - mpd_ssize_t q, r; - int k; - - if (mpd_isspecial(a) || mpd_isnegative(a) || a->exp != 0) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - digits = (a->digits < ctx->prec) ? ctx->prec : a->digits; - _mpd_idiv_word(&q, &r, digits, MPD_RDIGITS); - len = (r == 0) ? q : q+1; - if (!mpd_qresize(result, len, status)) { - return; - } - - for (i = 0; i < len; i++) { - x = (i < a->len) ? a->data[i] : 0; - z = 0; - for (k = 0; k < MPD_RDIGITS; k++) { - xbit = x % 10; - x /= 10; - if (xbit > 1) { - goto invalid_operation; - } - z += !xbit ? mpd_pow10[k] : 0; - } - result->data[i] = z; - } - - mpd_clear_flags(result); - result->exp = 0; - result->len = _mpd_real_size(result->data, len); - mpd_qresize(result, result->len, status); - mpd_setdigits(result); - _mpd_cap(result, ctx); - return; - -invalid_operation: - mpd_seterror(result, MPD_Invalid_operation, status); -} - -/* Exponent of the magnitude of the most significant digit of the operand. */ -void -mpd_qlogb(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - mpd_setspecial(result, MPD_POS, MPD_INF); - } - else if (mpd_iszerocoeff(a)) { - mpd_setspecial(result, MPD_NEG, MPD_INF); - *status |= MPD_Division_by_zero; - } - else { - mpd_qset_ssize(result, mpd_adjexp(a), ctx, status); - } -} - -/* Logical Or */ -void -mpd_qor(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - const mpd_t *big = a, *small = b; - mpd_uint_t x, y, z, xbit, ybit; - int k, mswdigits; - mpd_ssize_t i; - - if (mpd_isspecial(a) || mpd_isspecial(b) || - mpd_isnegative(a) || mpd_isnegative(b) || - a->exp != 0 || b->exp != 0) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (b->digits > a->digits) { - big = b; - small = a; - } - if (!mpd_qresize(result, big->len, status)) { - return; - } - - - /* full words */ - for (i = 0; i < small->len-1; i++) { - x = small->data[i]; - y = big->data[i]; - z = 0; - for (k = 0; k < MPD_RDIGITS; k++) { - xbit = x % 10; - x /= 10; - ybit = y % 10; - y /= 10; - if (xbit > 1 || ybit > 1) { - goto invalid_operation; - } - z += (xbit|ybit) ? mpd_pow10[k] : 0; - } - result->data[i] = z; - } - /* most significant word of small */ - x = small->data[i]; - y = big->data[i]; - z = 0; - mswdigits = mpd_word_digits(x); - for (k = 0; k < mswdigits; k++) { - xbit = x % 10; - x /= 10; - ybit = y % 10; - y /= 10; - if (xbit > 1 || ybit > 1) { - goto invalid_operation; - } - z += (xbit|ybit) ? mpd_pow10[k] : 0; - } - - /* scan for digits > 1 and copy the rest of y */ - for (; k < MPD_RDIGITS; k++) { - ybit = y % 10; - y /= 10; - if (ybit > 1) { - goto invalid_operation; - } - z += ybit*mpd_pow10[k]; - } - result->data[i++] = z; - /* scan for digits > 1 and copy the rest of big */ - for (; i < big->len; i++) { - y = big->data[i]; - for (k = 0; k < MPD_RDIGITS; k++) { - ybit = y % 10; - y /= 10; - if (ybit > 1) { - goto invalid_operation; - } - } - result->data[i] = big->data[i]; - } - - mpd_clear_flags(result); - result->exp = 0; - result->len = _mpd_real_size(result->data, big->len); - mpd_qresize(result, result->len, status); - mpd_setdigits(result); - _mpd_cap(result, ctx); - return; - -invalid_operation: - mpd_seterror(result, MPD_Invalid_operation, status); -} - -/* - * Rotate the coefficient of 'a' by 'b' digits. 'b' must be an integer with - * exponent 0. - */ -void -mpd_qrotate(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - MPD_NEW_STATIC(tmp,0,0,0,0); - MPD_NEW_STATIC(big,0,0,0,0); - MPD_NEW_STATIC(small,0,0,0,0); - mpd_ssize_t n, lshift, rshift; - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - } - if (b->exp != 0 || mpd_isinfinite(b)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - n = mpd_qget_ssize(b, &workstatus); - if (workstatus&MPD_Invalid_operation) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (n > ctx->prec || n < -ctx->prec) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_isinfinite(a)) { - mpd_qcopy(result, a, status); - return; - } - - if (n >= 0) { - lshift = n; - rshift = ctx->prec-n; - } - else { - lshift = ctx->prec+n; - rshift = -n; - } - - if (a->digits > ctx->prec) { - if (!mpd_qcopy(&tmp, a, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - goto finish; - } - _mpd_cap(&tmp, ctx); - a = &tmp; - } - - if (!mpd_qshiftl(&big, a, lshift, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - goto finish; - } - _mpd_cap(&big, ctx); - - if (mpd_qshiftr(&small, a, rshift, status) == MPD_UINT_MAX) { - mpd_seterror(result, MPD_Malloc_error, status); - goto finish; - } - _mpd_qadd(result, &big, &small, ctx, status); - - -finish: - mpd_del(&tmp); - mpd_del(&big); - mpd_del(&small); -} - -/* - * b must be an integer with exponent 0 and in the range +-2*(emax + prec). - * XXX: In my opinion +-(2*emax + prec) would be more sensible. - * The result is a with the value of b added to its exponent. - */ -void -mpd_qscaleb(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_uint_t n, maxjump; -#ifndef LEGACY_COMPILER - int64_t exp; -#else - mpd_uint_t x; - int x_sign, n_sign; - mpd_ssize_t exp; -#endif - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - } - if (b->exp != 0 || mpd_isinfinite(b)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - n = mpd_qabs_uint(b, &workstatus); - /* the spec demands this */ - maxjump = 2 * (mpd_uint_t)(ctx->emax + ctx->prec); - - if (n > maxjump || workstatus&MPD_Invalid_operation) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_isinfinite(a)) { - mpd_qcopy(result, a, status); - return; - } - -#ifndef LEGACY_COMPILER - exp = a->exp + (int64_t)n * mpd_arith_sign(b); - exp = (exp > MPD_EXP_INF) ? MPD_EXP_INF : exp; - exp = (exp < MPD_EXP_CLAMP) ? MPD_EXP_CLAMP : exp; -#else - x = (a->exp < 0) ? -a->exp : a->exp; - x_sign = (a->exp < 0) ? 1 : 0; - n_sign = mpd_isnegative(b) ? 1 : 0; - - if (x_sign == n_sign) { - x = x + n; - if (x < n) x = MPD_UINT_MAX; - } - else { - x_sign = (x >= n) ? x_sign : n_sign; - x = (x >= n) ? x - n : n - x; - } - if (!x_sign && x > MPD_EXP_INF) x = MPD_EXP_INF; - if (x_sign && x > -MPD_EXP_CLAMP) x = -MPD_EXP_CLAMP; - exp = x_sign ? -((mpd_ssize_t)x) : (mpd_ssize_t)x; -#endif - - mpd_qcopy(result, a, status); - result->exp = (mpd_ssize_t)exp; - - mpd_qfinalize(result, ctx, status); -} - -/* - * Shift the coefficient by n digits, positive n is a left shift. In the case - * of a left shift, the result is decapitated to fit the context precision. If - * you don't want that, use mpd_shiftl(). - */ -void -mpd_qshiftn(mpd_t *result, const mpd_t *a, mpd_ssize_t n, const mpd_context_t *ctx, - uint32_t *status) -{ - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - mpd_qcopy(result, a, status); - return; - } - - if (n >= 0 && n <= ctx->prec) { - mpd_qshiftl(result, a, n, status); - _mpd_cap(result, ctx); - } - else if (n < 0 && n >= -ctx->prec) { - if (!mpd_qcopy(result, a, status)) { - return; - } - _mpd_cap(result, ctx); - mpd_qshiftr_inplace(result, -n); - } - else { - mpd_seterror(result, MPD_Invalid_operation, status); - } -} - -/* - * Same as mpd_shiftn(), but the shift is specified by the decimal b, which - * must be an integer with a zero exponent. Infinities remain infinities. - */ -void -mpd_qshift(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, - uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_ssize_t n; - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - } - if (b->exp != 0 || mpd_isinfinite(b)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - n = mpd_qget_ssize(b, &workstatus); - if (workstatus&MPD_Invalid_operation) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (n > ctx->prec || n < -ctx->prec) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_isinfinite(a)) { - mpd_qcopy(result, a, status); - return; - } - - if (n >= 0) { - mpd_qshiftl(result, a, n, status); - _mpd_cap(result, ctx); - } - else { - if (!mpd_qcopy(result, a, status)) { - return; - } - _mpd_cap(result, ctx); - mpd_qshiftr_inplace(result, -n); - } -} - -/* Logical Xor */ -void -mpd_qxor(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - const mpd_t *big = a, *small = b; - mpd_uint_t x, y, z, xbit, ybit; - int k, mswdigits; - mpd_ssize_t i; - - if (mpd_isspecial(a) || mpd_isspecial(b) || - mpd_isnegative(a) || mpd_isnegative(b) || - a->exp != 0 || b->exp != 0) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (b->digits > a->digits) { - big = b; - small = a; - } - if (!mpd_qresize(result, big->len, status)) { - return; - } - - - /* full words */ - for (i = 0; i < small->len-1; i++) { - x = small->data[i]; - y = big->data[i]; - z = 0; - for (k = 0; k < MPD_RDIGITS; k++) { - xbit = x % 10; - x /= 10; - ybit = y % 10; - y /= 10; - if (xbit > 1 || ybit > 1) { - goto invalid_operation; - } - z += (xbit^ybit) ? mpd_pow10[k] : 0; - } - result->data[i] = z; - } - /* most significant word of small */ - x = small->data[i]; - y = big->data[i]; - z = 0; - mswdigits = mpd_word_digits(x); - for (k = 0; k < mswdigits; k++) { - xbit = x % 10; - x /= 10; - ybit = y % 10; - y /= 10; - if (xbit > 1 || ybit > 1) { - goto invalid_operation; - } - z += (xbit^ybit) ? mpd_pow10[k] : 0; - } - - /* scan for digits > 1 and copy the rest of y */ - for (; k < MPD_RDIGITS; k++) { - ybit = y % 10; - y /= 10; - if (ybit > 1) { - goto invalid_operation; - } - z += ybit*mpd_pow10[k]; - } - result->data[i++] = z; - /* scan for digits > 1 and copy the rest of big */ - for (; i < big->len; i++) { - y = big->data[i]; - for (k = 0; k < MPD_RDIGITS; k++) { - ybit = y % 10; - y /= 10; - if (ybit > 1) { - goto invalid_operation; - } - } - result->data[i] = big->data[i]; - } - - mpd_clear_flags(result); - result->exp = 0; - result->len = _mpd_real_size(result->data, big->len); - mpd_qresize(result, result->len, status); - mpd_setdigits(result); - _mpd_cap(result, ctx); - return; - -invalid_operation: - mpd_seterror(result, MPD_Invalid_operation, status); -} - - -/******************************************************************************/ -/* Arithmetic operations */ -/******************************************************************************/ - -/* - * The absolute value of a. If a is negative, the result is the same - * as the result of the minus operation. Otherwise, the result is the - * result of the plus operation. - */ -void -mpd_qabs(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - } - - if (mpd_isnegative(a)) { - mpd_qminus(result, a, ctx, status); - } - else { - mpd_qplus(result, a, ctx, status); - } -} - -static inline void -_mpd_ptrswap(const mpd_t **a, const mpd_t **b) -{ - const mpd_t *t = *a; - *a = *b; - *b = t; -} - -/* Add or subtract infinities. */ -static void -_mpd_qaddsub_inf(mpd_t *result, const mpd_t *a, const mpd_t *b, uint8_t sign_b, - uint32_t *status) -{ - if (mpd_isinfinite(a)) { - if (mpd_sign(a) != sign_b && mpd_isinfinite(b)) { - mpd_seterror(result, MPD_Invalid_operation, status); - } - else { - mpd_setspecial(result, mpd_sign(a), MPD_INF); - } - return; - } - assert(mpd_isinfinite(b)); - mpd_setspecial(result, sign_b, MPD_INF); -} - -/* Add or subtract non-special numbers. */ -static void -_mpd_qaddsub(mpd_t *result, const mpd_t *a, const mpd_t *b, uint8_t sign_b, - const mpd_context_t *ctx, uint32_t *status) -{ - const mpd_t *big, *small; - MPD_NEW_STATIC(big_aligned,0,0,0,0); - MPD_NEW_CONST(tiny,0,0,1,1,1,1); - mpd_uint_t carry; - mpd_ssize_t newsize, shift; - mpd_ssize_t exp, i; - int swap = 0; - - - /* compare exponents */ - big = a; small = b; - if (big->exp != small->exp) { - if (small->exp > big->exp) { - _mpd_ptrswap(&big, &small); - swap++; - } - /* align the coefficients */ - if (!mpd_iszerocoeff(big)) { - exp = big->exp - 1; - exp += (big->digits > ctx->prec) ? 0 : big->digits-ctx->prec-1; - if (mpd_adjexp(small) < exp) { - /* - * Avoid huge shifts by substituting a value for small that is - * guaranteed to produce the same results. - * - * adjexp(small) < exp if and only if: - * - * bdigits <= prec AND - * bdigits+shift >= prec+2+sdigits AND - * exp = bexp+bdigits-prec-2 - * - * 1234567000000000 -> bdigits + shift - * ----------XX1234 -> sdigits - * ----------X1 -> tiny-digits - * |- prec -| - * - * OR - * - * bdigits > prec AND - * shift > sdigits AND - * exp = bexp-1 - * - * 1234567892100000 -> bdigits + shift - * ----------XX1234 -> sdigits - * ----------X1 -> tiny-digits - * |- prec -| - * - * If tiny is zero, adding or subtracting is a no-op. - * Otherwise, adding tiny generates a non-zero digit either - * below the rounding digit or the least significant digit - * of big. When subtracting, tiny is in the same position as - * the carry that would be generated by subtracting sdigits. - */ - mpd_copy_flags(&tiny, small); - tiny.exp = exp; - tiny.digits = 1; - tiny.len = 1; - tiny.data[0] = mpd_iszerocoeff(small) ? 0 : 1; - small = &tiny; - } - /* This cannot wrap: the difference is positive and <= maxprec */ - shift = big->exp - small->exp; - if (!mpd_qshiftl(&big_aligned, big, shift, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - goto finish; - } - big = &big_aligned; - } - } - result->exp = small->exp; - - - /* compare length of coefficients */ - if (big->len < small->len) { - _mpd_ptrswap(&big, &small); - swap++; - } - - newsize = big->len; - if (!mpd_qresize(result, newsize, status)) { - goto finish; - } - - if (mpd_sign(a) == sign_b) { - - carry = _mpd_baseadd(result->data, big->data, small->data, - big->len, small->len); - - if (carry) { - newsize = big->len + 1; - if (!mpd_qresize(result, newsize, status)) { - goto finish; - } - result->data[newsize-1] = carry; - } - - result->len = newsize; - mpd_set_flags(result, sign_b); - } - else { - if (big->len == small->len) { - for (i=big->len-1; i >= 0; --i) { - if (big->data[i] != small->data[i]) { - if (big->data[i] < small->data[i]) { - _mpd_ptrswap(&big, &small); - swap++; - } - break; - } - } - } - - _mpd_basesub(result->data, big->data, small->data, - big->len, small->len); - newsize = _mpd_real_size(result->data, big->len); - /* resize to smaller cannot fail */ - (void)mpd_qresize(result, newsize, status); - - result->len = newsize; - sign_b = (swap & 1) ? sign_b : mpd_sign(a); - mpd_set_flags(result, sign_b); - - if (mpd_iszerocoeff(result)) { - mpd_set_positive(result); - if (ctx->round == MPD_ROUND_FLOOR) { - mpd_set_negative(result); - } - } - } - - mpd_setdigits(result); - -finish: - mpd_del(&big_aligned); -} - -/* Add a and b. No specials, no finalizing. */ -static void -_mpd_qadd(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - _mpd_qaddsub(result, a, b, mpd_sign(b), ctx, status); -} - -/* Subtract b from a. No specials, no finalizing. */ -static void -_mpd_qsub(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - _mpd_qaddsub(result, a, b, !mpd_sign(b), ctx, status); -} - -/* Add a and b. */ -void -mpd_qadd(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - _mpd_qaddsub_inf(result, a, b, mpd_sign(b), status); - return; - } - - _mpd_qaddsub(result, a, b, mpd_sign(b), ctx, status); - mpd_qfinalize(result, ctx, status); -} - -/* Add a and b. Set NaN/Invalid_operation if the result is inexact. */ -static void -_mpd_qadd_exact(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - - mpd_qadd(result, a, b, ctx, &workstatus); - *status |= workstatus; - if (workstatus & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { - mpd_seterror(result, MPD_Invalid_operation, status); - } -} - -/* Subtract b from a. */ -void -mpd_qsub(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - _mpd_qaddsub_inf(result, a, b, !mpd_sign(b), status); - return; - } - - _mpd_qaddsub(result, a, b, !mpd_sign(b), ctx, status); - mpd_qfinalize(result, ctx, status); -} - -/* Subtract b from a. Set NaN/Invalid_operation if the result is inexact. */ -static void -_mpd_qsub_exact(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - - mpd_qsub(result, a, b, ctx, &workstatus); - *status |= workstatus; - if (workstatus & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { - mpd_seterror(result, MPD_Invalid_operation, status); - } -} - -/* Add decimal and mpd_ssize_t. */ -void -mpd_qadd_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qsset_ssize(&bb, b, &maxcontext, status); - mpd_qadd(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Add decimal and mpd_uint_t. */ -void -mpd_qadd_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qsset_uint(&bb, b, &maxcontext, status); - mpd_qadd(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Subtract mpd_ssize_t from decimal. */ -void -mpd_qsub_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qsset_ssize(&bb, b, &maxcontext, status); - mpd_qsub(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Subtract mpd_uint_t from decimal. */ -void -mpd_qsub_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qsset_uint(&bb, b, &maxcontext, status); - mpd_qsub(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Add decimal and int32_t. */ -void -mpd_qadd_i32(mpd_t *result, const mpd_t *a, int32_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qadd_ssize(result, a, b, ctx, status); -} - -/* Add decimal and uint32_t. */ -void -mpd_qadd_u32(mpd_t *result, const mpd_t *a, uint32_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qadd_uint(result, a, b, ctx, status); -} - -#ifdef CONFIG_64 -/* Add decimal and int64_t. */ -void -mpd_qadd_i64(mpd_t *result, const mpd_t *a, int64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qadd_ssize(result, a, b, ctx, status); -} - -/* Add decimal and uint64_t. */ -void -mpd_qadd_u64(mpd_t *result, const mpd_t *a, uint64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qadd_uint(result, a, b, ctx, status); -} -#elif !defined(LEGACY_COMPILER) -/* Add decimal and int64_t. */ -void -mpd_qadd_i64(mpd_t *result, const mpd_t *a, int64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qset_i64(&bb, b, &maxcontext, status); - mpd_qadd(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Add decimal and uint64_t. */ -void -mpd_qadd_u64(mpd_t *result, const mpd_t *a, uint64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qset_u64(&bb, b, &maxcontext, status); - mpd_qadd(result, a, &bb, ctx, status); - mpd_del(&bb); -} -#endif - -/* Subtract int32_t from decimal. */ -void -mpd_qsub_i32(mpd_t *result, const mpd_t *a, int32_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qsub_ssize(result, a, b, ctx, status); -} - -/* Subtract uint32_t from decimal. */ -void -mpd_qsub_u32(mpd_t *result, const mpd_t *a, uint32_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qsub_uint(result, a, b, ctx, status); -} - -#ifdef CONFIG_64 -/* Subtract int64_t from decimal. */ -void -mpd_qsub_i64(mpd_t *result, const mpd_t *a, int64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qsub_ssize(result, a, b, ctx, status); -} - -/* Subtract uint64_t from decimal. */ -void -mpd_qsub_u64(mpd_t *result, const mpd_t *a, uint64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qsub_uint(result, a, b, ctx, status); -} -#elif !defined(LEGACY_COMPILER) -/* Subtract int64_t from decimal. */ -void -mpd_qsub_i64(mpd_t *result, const mpd_t *a, int64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qset_i64(&bb, b, &maxcontext, status); - mpd_qsub(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Subtract uint64_t from decimal. */ -void -mpd_qsub_u64(mpd_t *result, const mpd_t *a, uint64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qset_u64(&bb, b, &maxcontext, status); - mpd_qsub(result, a, &bb, ctx, status); - mpd_del(&bb); -} -#endif - - -/* Divide infinities. */ -static void -_mpd_qdiv_inf(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - if (mpd_isinfinite(a)) { - if (mpd_isinfinite(b)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - mpd_setspecial(result, mpd_sign(a)^mpd_sign(b), MPD_INF); - return; - } - assert(mpd_isinfinite(b)); - _settriple(result, mpd_sign(a)^mpd_sign(b), 0, mpd_etiny(ctx)); - *status |= MPD_Clamped; -} - -enum {NO_IDEAL_EXP, SET_IDEAL_EXP}; -/* Divide a by b. */ -static void -_mpd_qdiv(int action, mpd_t *q, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - MPD_NEW_STATIC(aligned,0,0,0,0); - mpd_uint_t ld; - mpd_ssize_t shift, exp, tz; - mpd_ssize_t newsize; - mpd_ssize_t ideal_exp; - mpd_uint_t rem; - uint8_t sign_a = mpd_sign(a); - uint8_t sign_b = mpd_sign(b); - - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(q, a, b, ctx, status)) { - return; - } - _mpd_qdiv_inf(q, a, b, ctx, status); - return; - } - if (mpd_iszerocoeff(b)) { - if (mpd_iszerocoeff(a)) { - mpd_seterror(q, MPD_Division_undefined, status); - } - else { - mpd_setspecial(q, sign_a^sign_b, MPD_INF); - *status |= MPD_Division_by_zero; - } - return; - } - if (mpd_iszerocoeff(a)) { - exp = a->exp - b->exp; - _settriple(q, sign_a^sign_b, 0, exp); - mpd_qfinalize(q, ctx, status); - return; - } - - shift = (b->digits - a->digits) + ctx->prec + 1; - ideal_exp = a->exp - b->exp; - exp = ideal_exp - shift; - if (shift > 0) { - if (!mpd_qshiftl(&aligned, a, shift, status)) { - mpd_seterror(q, MPD_Malloc_error, status); - goto finish; - } - a = &aligned; - } - else if (shift < 0) { - shift = -shift; - if (!mpd_qshiftl(&aligned, b, shift, status)) { - mpd_seterror(q, MPD_Malloc_error, status); - goto finish; - } - b = &aligned; - } - - - newsize = a->len - b->len + 1; - if ((q != b && q != a) || (q == b && newsize > b->len)) { - if (!mpd_qresize(q, newsize, status)) { - mpd_seterror(q, MPD_Malloc_error, status); - goto finish; - } - } - - - if (b->len == 1) { - rem = _mpd_shortdiv(q->data, a->data, a->len, b->data[0]); - } - else if (b->len <= MPD_NEWTONDIV_CUTOFF) { - int ret = _mpd_basedivmod(q->data, NULL, a->data, b->data, - a->len, b->len); - if (ret < 0) { - mpd_seterror(q, MPD_Malloc_error, status); - goto finish; - } - rem = ret; - } - else { - MPD_NEW_STATIC(r,0,0,0,0); - _mpd_base_ndivmod(q, &r, a, b, status); - if (mpd_isspecial(q) || mpd_isspecial(&r)) { - mpd_setspecial(q, MPD_POS, MPD_NAN); - mpd_del(&r); - goto finish; - } - rem = !mpd_iszerocoeff(&r); - mpd_del(&r); - newsize = q->len; - } - - newsize = _mpd_real_size(q->data, newsize); - /* resize to smaller cannot fail */ - mpd_qresize(q, newsize, status); - mpd_set_flags(q, sign_a^sign_b); - q->len = newsize; - mpd_setdigits(q); - - shift = ideal_exp - exp; - if (rem) { - ld = mpd_lsd(q->data[0]); - if (ld == 0 || ld == 5) { - q->data[0] += 1; - } - } - else if (action == SET_IDEAL_EXP && shift > 0) { - tz = mpd_trail_zeros(q); - shift = (tz > shift) ? shift : tz; - mpd_qshiftr_inplace(q, shift); - exp += shift; - } - - q->exp = exp; - - -finish: - mpd_del(&aligned); - mpd_qfinalize(q, ctx, status); -} - -/* Divide a by b. */ -void -mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - MPD_NEW_STATIC(aa,0,0,0,0); - MPD_NEW_STATIC(bb,0,0,0,0); - uint32_t xstatus = 0; - - if (q == a) { - if (!mpd_qcopy(&aa, a, status)) { - mpd_seterror(q, MPD_Malloc_error, status); - goto out; - } - a = &aa; - } - - if (q == b) { - if (!mpd_qcopy(&bb, b, status)) { - mpd_seterror(q, MPD_Malloc_error, status); - goto out; - } - b = &bb; - } - - _mpd_qdiv(SET_IDEAL_EXP, q, a, b, ctx, &xstatus); - - if (xstatus & (MPD_Malloc_error|MPD_Division_impossible)) { - /* Inexact quotients (the usual case) fill the entire context precision, - * which can lead to the above errors for very high precisions. Retry - * the operation with a lower precision in case the result is exact. - * - * We need an upper bound for the number of digits of a_coeff / b_coeff - * when the result is exact. If a_coeff' * 1 / b_coeff' is in lowest - * terms, then maxdigits(a_coeff') + maxdigits(1 / b_coeff') is a suitable - * bound. - * - * 1 / b_coeff' is exact iff b_coeff' exclusively has prime factors 2 or 5. - * The largest amount of digits is generated if b_coeff' is a power of 2 or - * a power of 5 and is less than or equal to log5(b_coeff') <= log2(b_coeff'). - * - * We arrive at a total upper bound: - * - * maxdigits(a_coeff') + maxdigits(1 / b_coeff') <= - * log10(a_coeff) + log2(b_coeff) = - * log10(a_coeff) + log10(b_coeff) / log10(2) <= - * a->digits + b->digits * 4; - */ - mpd_context_t workctx = *ctx; - uint32_t ystatus = 0; - - workctx.prec = a->digits + b->digits * 4; - if (workctx.prec >= ctx->prec) { - *status |= (xstatus&MPD_Errors); - goto out; /* No point in retrying, keep the original error. */ - } - - _mpd_qdiv(SET_IDEAL_EXP, q, a, b, &workctx, &ystatus); - if (ystatus != 0) { - ystatus = *status | ((ystatus|xstatus)&MPD_Errors); - mpd_seterror(q, ystatus, status); - } - } - else { - *status |= xstatus; - } - - -out: - mpd_del(&aa); - mpd_del(&bb); -} - -/* Internal function. */ -static void -_mpd_qdivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - MPD_NEW_STATIC(aligned,0,0,0,0); - mpd_ssize_t qsize, rsize; - mpd_ssize_t ideal_exp, expdiff, shift; - uint8_t sign_a = mpd_sign(a); - uint8_t sign_ab = mpd_sign(a)^mpd_sign(b); - - - ideal_exp = (a->exp > b->exp) ? b->exp : a->exp; - if (mpd_iszerocoeff(a)) { - if (!mpd_qcopy(r, a, status)) { - goto nanresult; /* GCOV_NOT_REACHED */ - } - r->exp = ideal_exp; - _settriple(q, sign_ab, 0, 0); - return; - } - - expdiff = mpd_adjexp(a) - mpd_adjexp(b); - if (expdiff < 0) { - if (a->exp > b->exp) { - /* positive and less than b->digits - a->digits */ - shift = a->exp - b->exp; - if (!mpd_qshiftl(r, a, shift, status)) { - goto nanresult; - } - r->exp = ideal_exp; - } - else { - if (!mpd_qcopy(r, a, status)) { - goto nanresult; - } - } - _settriple(q, sign_ab, 0, 0); - return; - } - if (expdiff > ctx->prec) { - *status |= MPD_Division_impossible; - goto nanresult; - } - - - /* - * At this point we have: - * (1) 0 <= a->exp + a->digits - b->exp - b->digits <= prec - * (2) a->exp - b->exp >= b->digits - a->digits - * (3) a->exp - b->exp <= prec + b->digits - a->digits - */ - if (a->exp != b->exp) { - shift = a->exp - b->exp; - if (shift > 0) { - /* by (3), after the shift a->digits <= prec + b->digits */ - if (!mpd_qshiftl(&aligned, a, shift, status)) { - goto nanresult; - } - a = &aligned; - } - else { - shift = -shift; - /* by (2), after the shift b->digits <= a->digits */ - if (!mpd_qshiftl(&aligned, b, shift, status)) { - goto nanresult; - } - b = &aligned; - } - } - - - qsize = a->len - b->len + 1; - if (!(q == a && qsize < a->len) && !(q == b && qsize < b->len)) { - if (!mpd_qresize(q, qsize, status)) { - goto nanresult; - } - } - - rsize = b->len; - if (!(r == a && rsize < a->len)) { - if (!mpd_qresize(r, rsize, status)) { - goto nanresult; - } - } - - if (b->len == 1) { - assert(b->data[0] != 0); /* annotation for scan-build */ - if (a->len == 1) { - _mpd_div_word(&q->data[0], &r->data[0], a->data[0], b->data[0]); - } - else { - r->data[0] = _mpd_shortdiv(q->data, a->data, a->len, b->data[0]); - } - } - else if (b->len <= MPD_NEWTONDIV_CUTOFF) { - int ret; - ret = _mpd_basedivmod(q->data, r->data, a->data, b->data, - a->len, b->len); - if (ret == -1) { - *status |= MPD_Malloc_error; - goto nanresult; - } - } - else { - _mpd_base_ndivmod(q, r, a, b, status); - if (mpd_isspecial(q) || mpd_isspecial(r)) { - goto nanresult; - } - qsize = q->len; - rsize = r->len; - } - - qsize = _mpd_real_size(q->data, qsize); - /* resize to smaller cannot fail */ - mpd_qresize(q, qsize, status); - q->len = qsize; - mpd_setdigits(q); - mpd_set_flags(q, sign_ab); - q->exp = 0; - if (q->digits > ctx->prec) { - *status |= MPD_Division_impossible; - goto nanresult; - } - - rsize = _mpd_real_size(r->data, rsize); - /* resize to smaller cannot fail */ - mpd_qresize(r, rsize, status); - r->len = rsize; - mpd_setdigits(r); - mpd_set_flags(r, sign_a); - r->exp = ideal_exp; - -out: - mpd_del(&aligned); - return; - -nanresult: - mpd_setspecial(q, MPD_POS, MPD_NAN); - mpd_setspecial(r, MPD_POS, MPD_NAN); - goto out; -} - -/* Integer division with remainder. */ -void -mpd_qdivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - uint8_t sign = mpd_sign(a)^mpd_sign(b); - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(q, a, b, ctx, status)) { - mpd_qcopy(r, q, status); - return; - } - if (mpd_isinfinite(a)) { - if (mpd_isinfinite(b)) { - mpd_setspecial(q, MPD_POS, MPD_NAN); - } - else { - mpd_setspecial(q, sign, MPD_INF); - } - mpd_setspecial(r, MPD_POS, MPD_NAN); - *status |= MPD_Invalid_operation; - return; - } - if (mpd_isinfinite(b)) { - if (!mpd_qcopy(r, a, status)) { - mpd_seterror(q, MPD_Malloc_error, status); - return; - } - mpd_qfinalize(r, ctx, status); - _settriple(q, sign, 0, 0); - return; - } - /* debug */ - abort(); /* GCOV_NOT_REACHED */ - } - if (mpd_iszerocoeff(b)) { - if (mpd_iszerocoeff(a)) { - mpd_setspecial(q, MPD_POS, MPD_NAN); - mpd_setspecial(r, MPD_POS, MPD_NAN); - *status |= MPD_Division_undefined; - } - else { - mpd_setspecial(q, sign, MPD_INF); - mpd_setspecial(r, MPD_POS, MPD_NAN); - *status |= (MPD_Division_by_zero|MPD_Invalid_operation); - } - return; - } - - _mpd_qdivmod(q, r, a, b, ctx, status); - mpd_qfinalize(q, ctx, status); - mpd_qfinalize(r, ctx, status); -} - -void -mpd_qdivint(mpd_t *q, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - MPD_NEW_STATIC(r,0,0,0,0); - uint8_t sign = mpd_sign(a)^mpd_sign(b); - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(q, a, b, ctx, status)) { - return; - } - if (mpd_isinfinite(a) && mpd_isinfinite(b)) { - mpd_seterror(q, MPD_Invalid_operation, status); - return; - } - if (mpd_isinfinite(a)) { - mpd_setspecial(q, sign, MPD_INF); - return; - } - if (mpd_isinfinite(b)) { - _settriple(q, sign, 0, 0); - return; - } - /* debug */ - abort(); /* GCOV_NOT_REACHED */ - } - if (mpd_iszerocoeff(b)) { - if (mpd_iszerocoeff(a)) { - mpd_seterror(q, MPD_Division_undefined, status); - } - else { - mpd_setspecial(q, sign, MPD_INF); - *status |= MPD_Division_by_zero; - } - return; - } - - - _mpd_qdivmod(q, &r, a, b, ctx, status); - mpd_del(&r); - mpd_qfinalize(q, ctx, status); -} - -/* Divide decimal by mpd_ssize_t. */ -void -mpd_qdiv_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qsset_ssize(&bb, b, &maxcontext, status); - mpd_qdiv(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Divide decimal by mpd_uint_t. */ -void -mpd_qdiv_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qsset_uint(&bb, b, &maxcontext, status); - mpd_qdiv(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Divide decimal by int32_t. */ -void -mpd_qdiv_i32(mpd_t *result, const mpd_t *a, int32_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qdiv_ssize(result, a, b, ctx, status); -} - -/* Divide decimal by uint32_t. */ -void -mpd_qdiv_u32(mpd_t *result, const mpd_t *a, uint32_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qdiv_uint(result, a, b, ctx, status); -} - -#ifdef CONFIG_64 -/* Divide decimal by int64_t. */ -void -mpd_qdiv_i64(mpd_t *result, const mpd_t *a, int64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qdiv_ssize(result, a, b, ctx, status); -} - -/* Divide decimal by uint64_t. */ -void -mpd_qdiv_u64(mpd_t *result, const mpd_t *a, uint64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qdiv_uint(result, a, b, ctx, status); -} -#elif !defined(LEGACY_COMPILER) -/* Divide decimal by int64_t. */ -void -mpd_qdiv_i64(mpd_t *result, const mpd_t *a, int64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qset_i64(&bb, b, &maxcontext, status); - mpd_qdiv(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Divide decimal by uint64_t. */ -void -mpd_qdiv_u64(mpd_t *result, const mpd_t *a, uint64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qset_u64(&bb, b, &maxcontext, status); - mpd_qdiv(result, a, &bb, ctx, status); - mpd_del(&bb); -} -#endif - -/* Pad the result with trailing zeros if it has fewer digits than prec. */ -static void -_mpd_zeropad(mpd_t *result, const mpd_context_t *ctx, uint32_t *status) -{ - if (!mpd_isspecial(result) && !mpd_iszero(result) && - result->digits < ctx->prec) { - mpd_ssize_t shift = ctx->prec - result->digits; - mpd_qshiftl(result, result, shift, status); - result->exp -= shift; - } -} - -/* Check if the result is guaranteed to be one. */ -static int -_mpd_qexp_check_one(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - MPD_NEW_CONST(lim,0,-(ctx->prec+1),1,1,1,9); - MPD_NEW_SHARED(aa, a); - - mpd_set_positive(&aa); - - /* abs(a) <= 9 * 10**(-prec-1) */ - if (_mpd_cmp(&aa, &lim) <= 0) { - _settriple(result, 0, 1, 0); - *status |= MPD_Rounded|MPD_Inexact; - return 1; - } - - return 0; -} - -/* - * Get the number of iterations for the Horner scheme in _mpd_qexp(). - */ -static inline mpd_ssize_t -_mpd_get_exp_iterations(const mpd_t *r, mpd_ssize_t p) -{ - mpd_ssize_t log10pbyr; /* lower bound for log10(p / abs(r)) */ - mpd_ssize_t n; - - assert(p >= 10); - assert(!mpd_iszero(r)); - assert(-p < mpd_adjexp(r) && mpd_adjexp(r) <= -1); - -#ifdef CONFIG_64 - if (p > (mpd_ssize_t)(1ULL<<52)) { - return MPD_SSIZE_MAX; - } -#endif - - /* - * Lower bound for log10(p / abs(r)): adjexp(p) - (adjexp(r) + 1) - * At this point (for CONFIG_64, CONFIG_32 is not problematic): - * 1) 10 <= p <= 2**52 - * 2) -p < adjexp(r) <= -1 - * 3) 1 <= log10pbyr <= 2**52 + 14 - */ - log10pbyr = (mpd_word_digits(p)-1) - (mpd_adjexp(r)+1); - - /* - * The numerator in the paper is 1.435 * p - 1.182, calculated - * exactly. We compensate for rounding errors by using 1.43503. - * ACL2 proofs: - * 1) exp-iter-approx-lower-bound: The term below evaluated - * in 53-bit floating point arithmetic is greater than or - * equal to the exact term used in the paper. - * 2) exp-iter-approx-upper-bound: The term below is less than - * or equal to 3/2 * p <= 3/2 * 2**52. - */ - n = (mpd_ssize_t)ceil((1.43503*(double)p - 1.182) / (double)log10pbyr); - return n >= 3 ? n : 3; -} - -/* - * Internal function, specials have been dealt with. Apart from Overflow - * and Underflow, two cases must be considered for the error of the result: - * - * 1) abs(a) <= 9 * 10**(-prec-1) ==> result == 1 - * - * Absolute error: abs(1 - e**x) < 10**(-prec) - * ------------------------------------------- - * - * 2) abs(a) > 9 * 10**(-prec-1) - * - * Relative error: abs(result - e**x) < 0.5 * 10**(-prec) * e**x - * ------------------------------------------------------------- - * - * The algorithm is from Hull&Abrham, Variable Precision Exponential Function, - * ACM Transactions on Mathematical Software, Vol. 12, No. 2, June 1986. - * - * Main differences: - * - * - The number of iterations for the Horner scheme is calculated using - * 53-bit floating point arithmetic. - * - * - In the error analysis for ER (relative error accumulated in the - * evaluation of the truncated series) the reduced operand r may - * have any number of digits. - * ACL2 proof: exponent-relative-error - * - * - The analysis for early abortion has been adapted for the mpd_t - * ranges. - */ -static void -_mpd_qexp(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx; - MPD_NEW_STATIC(tmp,0,0,0,0); - MPD_NEW_STATIC(sum,0,0,0,0); - MPD_NEW_CONST(word,0,0,1,1,1,1); - mpd_ssize_t j, n, t; - - assert(!mpd_isspecial(a)); - - if (mpd_iszerocoeff(a)) { - _settriple(result, MPD_POS, 1, 0); - return; - } - - /* - * We are calculating e^x = e^(r*10^t) = (e^r)^(10^t), where abs(r) < 1 and t >= 0. - * - * If t > 0, we have: - * - * (1) 0.1 <= r < 1, so e^0.1 <= e^r. If t > MAX_T, overflow occurs: - * - * MAX-EMAX+1 < log10(e^(0.1*10*t)) <= log10(e^(r*10^t)) < adjexp(e^(r*10^t))+1 - * - * (2) -1 < r <= -0.1, so e^r <= e^-0.1. If t > MAX_T, underflow occurs: - * - * adjexp(e^(r*10^t)) <= log10(e^(r*10^t)) <= log10(e^(-0.1*10^t)) < MIN-ETINY - */ -#if defined(CONFIG_64) - #define MPD_EXP_MAX_T 19 -#elif defined(CONFIG_32) - #define MPD_EXP_MAX_T 10 -#endif - t = a->digits + a->exp; - t = (t > 0) ? t : 0; - if (t > MPD_EXP_MAX_T) { - if (mpd_ispositive(a)) { - mpd_setspecial(result, MPD_POS, MPD_INF); - *status |= MPD_Overflow|MPD_Inexact|MPD_Rounded; - } - else { - _settriple(result, MPD_POS, 0, mpd_etiny(ctx)); - *status |= (MPD_Inexact|MPD_Rounded|MPD_Subnormal| - MPD_Underflow|MPD_Clamped); - } - return; - } - - /* abs(a) <= 9 * 10**(-prec-1) */ - if (_mpd_qexp_check_one(result, a, ctx, status)) { - return; - } - - mpd_maxcontext(&workctx); - workctx.prec = ctx->prec + t + 2; - workctx.prec = (workctx.prec < 10) ? 10 : workctx.prec; - workctx.round = MPD_ROUND_HALF_EVEN; - - if (!mpd_qcopy(result, a, status)) { - return; - } - result->exp -= t; - - /* - * At this point: - * 1) 9 * 10**(-prec-1) < abs(a) - * 2) 9 * 10**(-prec-t-1) < abs(r) - * 3) log10(9) - prec - t - 1 < log10(abs(r)) < adjexp(abs(r)) + 1 - * 4) - prec - t - 2 < adjexp(abs(r)) <= -1 - */ - n = _mpd_get_exp_iterations(result, workctx.prec); - if (n == MPD_SSIZE_MAX) { - mpd_seterror(result, MPD_Invalid_operation, status); /* GCOV_UNLIKELY */ - return; /* GCOV_UNLIKELY */ - } - - _settriple(&sum, MPD_POS, 1, 0); - - for (j = n-1; j >= 1; j--) { - word.data[0] = j; - mpd_setdigits(&word); - mpd_qdiv(&tmp, result, &word, &workctx, &workctx.status); - mpd_qfma(&sum, &sum, &tmp, &one, &workctx, &workctx.status); - } - -#ifdef CONFIG_64 - _mpd_qpow_uint(result, &sum, mpd_pow10[t], MPD_POS, &workctx, status); -#else - if (t <= MPD_MAX_POW10) { - _mpd_qpow_uint(result, &sum, mpd_pow10[t], MPD_POS, &workctx, status); - } - else { - t -= MPD_MAX_POW10; - _mpd_qpow_uint(&tmp, &sum, mpd_pow10[MPD_MAX_POW10], MPD_POS, - &workctx, status); - _mpd_qpow_uint(result, &tmp, mpd_pow10[t], MPD_POS, &workctx, status); - } -#endif - - mpd_del(&tmp); - mpd_del(&sum); - *status |= (workctx.status&MPD_Errors); - *status |= (MPD_Inexact|MPD_Rounded); -} - -/* exp(a) */ -void -mpd_qexp(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx; - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - if (mpd_isnegative(a)) { - _settriple(result, MPD_POS, 0, 0); - } - else { - mpd_setspecial(result, MPD_POS, MPD_INF); - } - return; - } - if (mpd_iszerocoeff(a)) { - _settriple(result, MPD_POS, 1, 0); - return; - } - - workctx = *ctx; - workctx.round = MPD_ROUND_HALF_EVEN; - - if (ctx->allcr) { - MPD_NEW_STATIC(t1, 0,0,0,0); - MPD_NEW_STATIC(t2, 0,0,0,0); - MPD_NEW_STATIC(ulp, 0,0,0,0); - MPD_NEW_STATIC(aa, 0,0,0,0); - mpd_ssize_t prec; - mpd_ssize_t ulpexp; - uint32_t workstatus; - - if (result == a) { - if (!mpd_qcopy(&aa, a, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - a = &aa; - } - - workctx.clamp = 0; - prec = ctx->prec + 3; - while (1) { - workctx.prec = prec; - workstatus = 0; - - _mpd_qexp(result, a, &workctx, &workstatus); - *status |= workstatus; - - ulpexp = result->exp + result->digits - workctx.prec; - if (workstatus & MPD_Underflow) { - /* The effective work precision is result->digits. */ - ulpexp = result->exp; - } - _ssettriple(&ulp, MPD_POS, 1, ulpexp); - - /* - * At this point [1]: - * 1) abs(result - e**x) < 0.5 * 10**(-prec) * e**x - * 2) result - ulp < e**x < result + ulp - * 3) result - ulp < result < result + ulp - * - * If round(result-ulp)==round(result+ulp), then - * round(result)==round(e**x). Therefore the result - * is correctly rounded. - * - * [1] If abs(a) <= 9 * 10**(-prec-1), use the absolute - * error for a similar argument. - */ - workctx.prec = ctx->prec; - mpd_qadd(&t1, result, &ulp, &workctx, &workctx.status); - mpd_qsub(&t2, result, &ulp, &workctx, &workctx.status); - if (mpd_isspecial(result) || mpd_iszerocoeff(result) || - mpd_qcmp(&t1, &t2, status) == 0) { - workctx.clamp = ctx->clamp; - _mpd_zeropad(result, &workctx, status); - mpd_check_underflow(result, &workctx, status); - mpd_qfinalize(result, &workctx, status); - break; - } - prec += MPD_RDIGITS; - } - mpd_del(&t1); - mpd_del(&t2); - mpd_del(&ulp); - mpd_del(&aa); - } - else { - _mpd_qexp(result, a, &workctx, status); - _mpd_zeropad(result, &workctx, status); - mpd_check_underflow(result, &workctx, status); - mpd_qfinalize(result, &workctx, status); - } -} - -/* Fused multiply-add: (a * b) + c, with a single final rounding. */ -void -mpd_qfma(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_t *cc = NULL; - - if (result == c) { - if ((cc = mpd_qncopy(c)) == NULL) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - c = cc; - } - - _mpd_qmul(result, a, b, ctx, &workstatus); - if (!(workstatus&MPD_Invalid_operation)) { - mpd_qadd(result, result, c, ctx, &workstatus); - } - - if (cc) mpd_del(cc); - *status |= workstatus; -} - -/* - * Schedule the optimal precision increase for the Newton iteration. - * v := input operand - * z_0 := initial approximation - * initprec := natural number such that abs(log(v) - z_0) < 10**-initprec - * maxprec := target precision - * - * For convenience the output klist contains the elements in reverse order: - * klist := [k_n-1, ..., k_0], where - * 1) k_0 <= initprec and - * 2) abs(log(v) - result) < 10**(-2*k_n-1 + 1) <= 10**-maxprec. - */ -static inline int -ln_schedule_prec(mpd_ssize_t klist[MPD_MAX_PREC_LOG2], mpd_ssize_t maxprec, - mpd_ssize_t initprec) -{ - mpd_ssize_t k; - int i; - - assert(maxprec >= 2 && initprec >= 2); - if (maxprec <= initprec) return -1; - - i = 0; k = maxprec; - do { - k = (k+2) / 2; - klist[i++] = k; - } while (k > initprec); - - return i-1; -} - -/* The constants have been verified with both decimal.py and mpfr. */ -#ifdef CONFIG_64 -#if MPD_RDIGITS != 19 - #error "mpdecimal.c: MPD_RDIGITS must be 19." -#endif -static const mpd_uint_t mpd_ln10_data[MPD_MINALLOC_MAX] = { - 6983716328982174407ULL, 9089704281976336583ULL, 1515961135648465461ULL, - 4416816335727555703ULL, 2900988039194170265ULL, 2307925037472986509ULL, - 107598438319191292ULL, 3466624107184669231ULL, 4450099781311469159ULL, - 9807828059751193854ULL, 7713456862091670584ULL, 1492198849978748873ULL, - 6528728696511086257ULL, 2385392051446341972ULL, 8692180205189339507ULL, - 6518769751037497088ULL, 2375253577097505395ULL, 9095610299291824318ULL, - 982748238504564801ULL, 5438635917781170543ULL, 7547331541421808427ULL, - 752371033310119785ULL, 3171643095059950878ULL, 9785265383207606726ULL, - 2932258279850258550ULL, 5497347726624257094ULL, 2976979522110718264ULL, - 9221477656763693866ULL, 1979650047149510504ULL, 6674183485704422507ULL, - 9702766860595249671ULL, 9278096762712757753ULL, 9314848524948644871ULL, - 6826928280848118428ULL, 754403708474699401ULL, 230105703089634572ULL, - 1929203337658714166ULL, 7589402567763113569ULL, 4208241314695689016ULL, - 2922455440575892572ULL, 9356734206705811364ULL, 2684916746550586856ULL, - 644507064800027750ULL, 9476834636167921018ULL, 5659121373450747856ULL, - 2835522011480466371ULL, 6470806855677432162ULL, 7141748003688084012ULL, - 9619404400222105101ULL, 5504893431493939147ULL, 6674744042432743651ULL, - 2287698219886746543ULL, 7773262884616336622ULL, 1985283935053089653ULL, - 4680843799894826233ULL, 8168948290720832555ULL, 8067566662873690987ULL, - 6248633409525465082ULL, 9829834196778404228ULL, 3524802359972050895ULL, - 3327900967572609677ULL, 110148862877297603ULL, 179914546843642076ULL, - 2302585092994045684ULL -}; -#else -#if MPD_RDIGITS != 9 - #error "mpdecimal.c: MPD_RDIGITS must be 9." -#endif -static const mpd_uint_t mpd_ln10_data[MPD_MINALLOC_MAX] = { - 401682692UL, 708474699UL, 720754403UL, 30896345UL, 602301057UL, 765871416UL, - 192920333UL, 763113569UL, 589402567UL, 956890167UL, 82413146UL, 589257242UL, - 245544057UL, 811364292UL, 734206705UL, 868569356UL, 167465505UL, 775026849UL, - 706480002UL, 18064450UL, 636167921UL, 569476834UL, 734507478UL, 156591213UL, - 148046637UL, 283552201UL, 677432162UL, 470806855UL, 880840126UL, 417480036UL, - 210510171UL, 940440022UL, 939147961UL, 893431493UL, 436515504UL, 440424327UL, - 654366747UL, 821988674UL, 622228769UL, 884616336UL, 537773262UL, 350530896UL, - 319852839UL, 989482623UL, 468084379UL, 720832555UL, 168948290UL, 736909878UL, - 675666628UL, 546508280UL, 863340952UL, 404228624UL, 834196778UL, 508959829UL, - 23599720UL, 967735248UL, 96757260UL, 603332790UL, 862877297UL, 760110148UL, - 468436420UL, 401799145UL, 299404568UL, 230258509UL -}; -#endif -/* _mpd_ln10 is used directly for precisions smaller than MINALLOC_MAX*RDIGITS. - Otherwise, it serves as the initial approximation for calculating ln(10). */ -static const mpd_t _mpd_ln10 = { - MPD_STATIC|MPD_CONST_DATA, -(MPD_MINALLOC_MAX*MPD_RDIGITS-1), - MPD_MINALLOC_MAX*MPD_RDIGITS, MPD_MINALLOC_MAX, MPD_MINALLOC_MAX, - (mpd_uint_t *)mpd_ln10_data -}; - -/* - * Set 'result' to log(10). - * Ulp error: abs(result - log(10)) < ulp(log(10)) - * Relative error: abs(result - log(10)) < 5 * 10**-prec * log(10) - * - * NOTE: The relative error is not derived from the ulp error, but - * calculated separately using the fact that 23/10 < log(10) < 24/10. - */ -void -mpd_qln10(mpd_t *result, mpd_ssize_t prec, uint32_t *status) -{ - mpd_context_t varcontext, maxcontext; - MPD_NEW_STATIC(tmp, 0,0,0,0); - MPD_NEW_CONST(static10, 0,0,2,1,1,10); - mpd_ssize_t klist[MPD_MAX_PREC_LOG2]; - mpd_uint_t rnd; - mpd_ssize_t shift; - int i; - - assert(prec >= 1); - - shift = MPD_MINALLOC_MAX*MPD_RDIGITS-prec; - shift = shift < 0 ? 0 : shift; - - rnd = mpd_qshiftr(result, &_mpd_ln10, shift, status); - if (rnd == MPD_UINT_MAX) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - result->exp = -(result->digits-1); - - mpd_maxcontext(&maxcontext); - if (prec < MPD_MINALLOC_MAX*MPD_RDIGITS) { - maxcontext.prec = prec; - _mpd_apply_round_excess(result, rnd, &maxcontext, status); - *status |= (MPD_Inexact|MPD_Rounded); - return; - } - - mpd_maxcontext(&varcontext); - varcontext.round = MPD_ROUND_TRUNC; - - i = ln_schedule_prec(klist, prec+2, -result->exp); - for (; i >= 0; i--) { - varcontext.prec = 2*klist[i]+3; - result->flags ^= MPD_NEG; - _mpd_qexp(&tmp, result, &varcontext, status); - result->flags ^= MPD_NEG; - mpd_qmul(&tmp, &static10, &tmp, &varcontext, status); - mpd_qsub(&tmp, &tmp, &one, &maxcontext, status); - mpd_qadd(result, result, &tmp, &maxcontext, status); - if (mpd_isspecial(result)) { - break; - } - } - - mpd_del(&tmp); - maxcontext.prec = prec; - mpd_qfinalize(result, &maxcontext, status); -} - -/* - * Initial approximations for the ln() iteration. The values have the - * following properties (established with both decimal.py and mpfr): - * - * Index 0 - 400, logarithms of x in [1.00, 5.00]: - * abs(lnapprox[i] * 10**-3 - log((i+100)/100)) < 10**-2 - * abs(lnapprox[i] * 10**-3 - log((i+1+100)/100)) < 10**-2 - * - * Index 401 - 899, logarithms of x in (0.500, 0.999]: - * abs(-lnapprox[i] * 10**-3 - log((i+100)/1000)) < 10**-2 - * abs(-lnapprox[i] * 10**-3 - log((i+1+100)/1000)) < 10**-2 - */ -static const uint16_t lnapprox[900] = { - /* index 0 - 400: log((i+100)/100) * 1000 */ - 0, 10, 20, 30, 39, 49, 58, 68, 77, 86, 95, 104, 113, 122, 131, 140, 148, 157, - 166, 174, 182, 191, 199, 207, 215, 223, 231, 239, 247, 255, 262, 270, 278, - 285, 293, 300, 308, 315, 322, 329, 336, 344, 351, 358, 365, 372, 378, 385, - 392, 399, 406, 412, 419, 425, 432, 438, 445, 451, 457, 464, 470, 476, 482, - 489, 495, 501, 507, 513, 519, 525, 531, 536, 542, 548, 554, 560, 565, 571, - 577, 582, 588, 593, 599, 604, 610, 615, 621, 626, 631, 637, 642, 647, 652, - 658, 663, 668, 673, 678, 683, 688, 693, 698, 703, 708, 713, 718, 723, 728, - 732, 737, 742, 747, 751, 756, 761, 766, 770, 775, 779, 784, 788, 793, 798, - 802, 806, 811, 815, 820, 824, 829, 833, 837, 842, 846, 850, 854, 859, 863, - 867, 871, 876, 880, 884, 888, 892, 896, 900, 904, 908, 912, 916, 920, 924, - 928, 932, 936, 940, 944, 948, 952, 956, 959, 963, 967, 971, 975, 978, 982, - 986, 990, 993, 997, 1001, 1004, 1008, 1012, 1015, 1019, 1022, 1026, 1030, - 1033, 1037, 1040, 1044, 1047, 1051, 1054, 1058, 1061, 1065, 1068, 1072, 1075, - 1078, 1082, 1085, 1089, 1092, 1095, 1099, 1102, 1105, 1109, 1112, 1115, 1118, - 1122, 1125, 1128, 1131, 1135, 1138, 1141, 1144, 1147, 1151, 1154, 1157, 1160, - 1163, 1166, 1169, 1172, 1176, 1179, 1182, 1185, 1188, 1191, 1194, 1197, 1200, - 1203, 1206, 1209, 1212, 1215, 1218, 1221, 1224, 1227, 1230, 1233, 1235, 1238, - 1241, 1244, 1247, 1250, 1253, 1256, 1258, 1261, 1264, 1267, 1270, 1273, 1275, - 1278, 1281, 1284, 1286, 1289, 1292, 1295, 1297, 1300, 1303, 1306, 1308, 1311, - 1314, 1316, 1319, 1322, 1324, 1327, 1330, 1332, 1335, 1338, 1340, 1343, 1345, - 1348, 1351, 1353, 1356, 1358, 1361, 1364, 1366, 1369, 1371, 1374, 1376, 1379, - 1381, 1384, 1386, 1389, 1391, 1394, 1396, 1399, 1401, 1404, 1406, 1409, 1411, - 1413, 1416, 1418, 1421, 1423, 1426, 1428, 1430, 1433, 1435, 1437, 1440, 1442, - 1445, 1447, 1449, 1452, 1454, 1456, 1459, 1461, 1463, 1466, 1468, 1470, 1472, - 1475, 1477, 1479, 1482, 1484, 1486, 1488, 1491, 1493, 1495, 1497, 1500, 1502, - 1504, 1506, 1509, 1511, 1513, 1515, 1517, 1520, 1522, 1524, 1526, 1528, 1530, - 1533, 1535, 1537, 1539, 1541, 1543, 1545, 1548, 1550, 1552, 1554, 1556, 1558, - 1560, 1562, 1564, 1567, 1569, 1571, 1573, 1575, 1577, 1579, 1581, 1583, 1585, - 1587, 1589, 1591, 1593, 1595, 1597, 1599, 1601, 1603, 1605, 1607, 1609, - /* index 401 - 899: -log((i+100)/1000) * 1000 */ - 691, 689, 687, 685, 683, 681, 679, 677, 675, 673, 671, 669, 668, 666, 664, - 662, 660, 658, 656, 654, 652, 650, 648, 646, 644, 642, 641, 639, 637, 635, - 633, 631, 629, 627, 626, 624, 622, 620, 618, 616, 614, 612, 611, 609, 607, - 605, 603, 602, 600, 598, 596, 594, 592, 591, 589, 587, 585, 583, 582, 580, - 578, 576, 574, 573, 571, 569, 567, 566, 564, 562, 560, 559, 557, 555, 553, - 552, 550, 548, 546, 545, 543, 541, 540, 538, 536, 534, 533, 531, 529, 528, - 526, 524, 523, 521, 519, 518, 516, 514, 512, 511, 509, 508, 506, 504, 502, - 501, 499, 498, 496, 494, 493, 491, 489, 488, 486, 484, 483, 481, 480, 478, - 476, 475, 473, 472, 470, 468, 467, 465, 464, 462, 460, 459, 457, 456, 454, - 453, 451, 449, 448, 446, 445, 443, 442, 440, 438, 437, 435, 434, 432, 431, - 429, 428, 426, 425, 423, 422, 420, 419, 417, 416, 414, 412, 411, 410, 408, - 406, 405, 404, 402, 400, 399, 398, 396, 394, 393, 392, 390, 389, 387, 386, - 384, 383, 381, 380, 378, 377, 375, 374, 372, 371, 370, 368, 367, 365, 364, - 362, 361, 360, 358, 357, 355, 354, 352, 351, 350, 348, 347, 345, 344, 342, - 341, 340, 338, 337, 336, 334, 333, 331, 330, 328, 327, 326, 324, 323, 322, - 320, 319, 318, 316, 315, 313, 312, 311, 309, 308, 306, 305, 304, 302, 301, - 300, 298, 297, 296, 294, 293, 292, 290, 289, 288, 286, 285, 284, 282, 281, - 280, 278, 277, 276, 274, 273, 272, 270, 269, 268, 267, 265, 264, 263, 261, - 260, 259, 258, 256, 255, 254, 252, 251, 250, 248, 247, 246, 245, 243, 242, - 241, 240, 238, 237, 236, 234, 233, 232, 231, 229, 228, 227, 226, 224, 223, - 222, 221, 219, 218, 217, 216, 214, 213, 212, 211, 210, 208, 207, 206, 205, - 203, 202, 201, 200, 198, 197, 196, 195, 194, 192, 191, 190, 189, 188, 186, - 185, 184, 183, 182, 180, 179, 178, 177, 176, 174, 173, 172, 171, 170, 168, - 167, 166, 165, 164, 162, 161, 160, 159, 158, 157, 156, 154, 153, 152, 151, - 150, 148, 147, 146, 145, 144, 143, 142, 140, 139, 138, 137, 136, 135, 134, - 132, 131, 130, 129, 128, 127, 126, 124, 123, 122, 121, 120, 119, 118, 116, - 115, 114, 113, 112, 111, 110, 109, 108, 106, 105, 104, 103, 102, 101, 100, - 99, 98, 97, 95, 94, 93, 92, 91, 90, 89, 88, 87, 86, 84, 83, 82, 81, 80, 79, - 78, 77, 76, 75, 74, 73, 72, 70, 69, 68, 67, 66, 65, 64, 63, 62, 61, 60, 59, - 58, 57, 56, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, - 38, 37, 36, 35, 34, 33, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, - 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 -}; - -/* - * Internal ln() function that does not check for specials, zero or one. - * Relative error: abs(result - log(a)) < 0.1 * 10**-prec * abs(log(a)) - */ -static void -_mpd_qln(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t varcontext, maxcontext; - mpd_t *z = result; - MPD_NEW_STATIC(v,0,0,0,0); - MPD_NEW_STATIC(vtmp,0,0,0,0); - MPD_NEW_STATIC(tmp,0,0,0,0); - mpd_ssize_t klist[MPD_MAX_PREC_LOG2]; - mpd_ssize_t maxprec, shift, t; - mpd_ssize_t a_digits, a_exp; - mpd_uint_t dummy, x; - int i; - - assert(!mpd_isspecial(a) && !mpd_iszerocoeff(a)); - - /* - * We are calculating ln(a) = ln(v * 10^t) = ln(v) + t*ln(10), - * where 0.5 < v <= 5. - */ - if (!mpd_qcopy(&v, a, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - goto finish; - } - - /* Initial approximation: we have at least one non-zero digit */ - _mpd_get_msdigits(&dummy, &x, &v, 3); - if (x < 10) x *= 10; - if (x < 100) x *= 10; - x -= 100; - - /* a may equal z */ - a_digits = a->digits; - a_exp = a->exp; - - mpd_minalloc(z); - mpd_clear_flags(z); - z->data[0] = lnapprox[x]; - z->len = 1; - z->exp = -3; - mpd_setdigits(z); - - if (x <= 400) { - /* Reduce the input operand to 1.00 <= v <= 5.00. Let y = x + 100, - * so 100 <= y <= 500. Since y contains the most significant digits - * of v, y/100 <= v < (y+1)/100 and abs(z - log(v)) < 10**-2. */ - v.exp = -(a_digits - 1); - t = a_exp + a_digits - 1; - } - else { - /* Reduce the input operand to 0.500 < v <= 0.999. Let y = x + 100, - * so 500 < y <= 999. Since y contains the most significant digits - * of v, y/1000 <= v < (y+1)/1000 and abs(z - log(v)) < 10**-2. */ - v.exp = -a_digits; - t = a_exp + a_digits; - mpd_set_negative(z); - } - - mpd_maxcontext(&maxcontext); - mpd_maxcontext(&varcontext); - varcontext.round = MPD_ROUND_TRUNC; - - maxprec = ctx->prec + 2; - if (t == 0 && (x <= 15 || x >= 800)) { - /* 0.900 <= v <= 1.15: Estimate the magnitude of the logarithm. - * If ln(v) will underflow, skip the loop. Otherwise, adjust the - * precision upwards in order to obtain a sufficient number of - * significant digits. - * - * Case v > 1: - * abs((v-1)/10) < abs((v-1)/v) < abs(ln(v)) < abs(v-1) - * Case v < 1: - * abs(v-1) < abs(ln(v)) < abs((v-1)/v) < abs((v-1)*10) - */ - int cmp = _mpd_cmp(&v, &one); - - /* Upper bound (assume v > 1): abs(v-1), unrounded */ - _mpd_qsub(&tmp, &v, &one, &maxcontext, &maxcontext.status); - if (maxcontext.status & MPD_Errors) { - mpd_seterror(result, MPD_Malloc_error, status); - goto finish; - } - - if (cmp < 0) { - /* v < 1: abs((v-1)*10) */ - tmp.exp += 1; - } - if (mpd_adjexp(&tmp) < mpd_etiny(ctx)) { - /* The upper bound is less than etiny: Underflow to zero */ - _settriple(result, (cmp<0), 1, mpd_etiny(ctx)-1); - goto finish; - } - /* Lower bound: abs((v-1)/10) or abs(v-1) */ - tmp.exp -= 1; - if (mpd_adjexp(&tmp) < 0) { - /* Absolute error of the loop: abs(z - log(v)) < 10**-p. If - * p = ctx->prec+2-adjexp(lower), then the relative error of - * the result is (using 10**adjexp(x) <= abs(x)): - * - * abs(z - log(v)) / abs(log(v)) < 10**-p / abs(log(v)) - * <= 10**(-ctx->prec-2) - */ - maxprec = maxprec - mpd_adjexp(&tmp); - } - } - - i = ln_schedule_prec(klist, maxprec, 2); - for (; i >= 0; i--) { - varcontext.prec = 2*klist[i]+3; - z->flags ^= MPD_NEG; - _mpd_qexp(&tmp, z, &varcontext, status); - z->flags ^= MPD_NEG; - - if (v.digits > varcontext.prec) { - shift = v.digits - varcontext.prec; - mpd_qshiftr(&vtmp, &v, shift, status); - vtmp.exp += shift; - mpd_qmul(&tmp, &vtmp, &tmp, &varcontext, status); - } - else { - mpd_qmul(&tmp, &v, &tmp, &varcontext, status); - } - - mpd_qsub(&tmp, &tmp, &one, &maxcontext, status); - mpd_qadd(z, z, &tmp, &maxcontext, status); - if (mpd_isspecial(z)) { - break; - } - } - - /* - * Case t == 0: - * t * log(10) == 0, the result does not change and the analysis - * above applies. If v < 0.900 or v > 1.15, the relative error is - * less than 10**(-ctx.prec-1). - * Case t != 0: - * z := approx(log(v)) - * y := approx(log(10)) - * p := maxprec = ctx->prec + 2 - * Absolute errors: - * 1) abs(z - log(v)) < 10**-p - * 2) abs(y - log(10)) < 10**-p - * The multiplication is exact, so: - * 3) abs(t*y - t*log(10)) < t*10**-p - * The sum is exact, so: - * 4) abs((z + t*y) - (log(v) + t*log(10))) < (abs(t) + 1) * 10**-p - * Bounds for log(v) and log(10): - * 5) -7/10 < log(v) < 17/10 - * 6) 23/10 < log(10) < 24/10 - * Using 4), 5), 6) and t != 0, the relative error is: - * - * 7) relerr < ((abs(t) + 1)*10**-p) / abs(log(v) + t*log(10)) - * < 0.5 * 10**(-p + 1) = 0.5 * 10**(-ctx->prec-1) - */ - mpd_qln10(&v, maxprec+1, status); - mpd_qmul_ssize(&tmp, &v, t, &maxcontext, status); - mpd_qadd(result, &tmp, z, &maxcontext, status); - - -finish: - *status |= (MPD_Inexact|MPD_Rounded); - mpd_del(&v); - mpd_del(&vtmp); - mpd_del(&tmp); -} - -/* ln(a) */ -void -mpd_qln(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx; - mpd_ssize_t adjexp, t; - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - if (mpd_isnegative(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - mpd_setspecial(result, MPD_POS, MPD_INF); - return; - } - if (mpd_iszerocoeff(a)) { - mpd_setspecial(result, MPD_NEG, MPD_INF); - return; - } - if (mpd_isnegative(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (_mpd_cmp(a, &one) == 0) { - _settriple(result, MPD_POS, 0, 0); - return; - } - /* - * Check if the result will overflow (0 < x, x != 1): - * 1) log10(x) < 0 iff adjexp(x) < 0 - * 2) 0 < x /\ x <= y ==> adjexp(x) <= adjexp(y) - * 3) 0 < x /\ x != 1 ==> 2 * abs(log10(x)) < abs(log(x)) - * 4) adjexp(x) <= log10(x) < adjexp(x) + 1 - * - * Case adjexp(x) >= 0: - * 5) 2 * adjexp(x) < abs(log(x)) - * Case adjexp(x) > 0: - * 6) adjexp(2 * adjexp(x)) <= adjexp(abs(log(x))) - * Case adjexp(x) == 0: - * mpd_exp_digits(t)-1 == 0 <= emax (the shortcut is not triggered) - * - * Case adjexp(x) < 0: - * 7) 2 * (-adjexp(x) - 1) < abs(log(x)) - * Case adjexp(x) < -1: - * 8) adjexp(2 * (-adjexp(x) - 1)) <= adjexp(abs(log(x))) - * Case adjexp(x) == -1: - * mpd_exp_digits(t)-1 == 0 <= emax (the shortcut is not triggered) - */ - adjexp = mpd_adjexp(a); - t = (adjexp < 0) ? -adjexp-1 : adjexp; - t *= 2; - if (mpd_exp_digits(t)-1 > ctx->emax) { - *status |= MPD_Overflow|MPD_Inexact|MPD_Rounded; - mpd_setspecial(result, (adjexp<0), MPD_INF); - return; - } - - workctx = *ctx; - workctx.round = MPD_ROUND_HALF_EVEN; - - if (ctx->allcr) { - MPD_NEW_STATIC(t1, 0,0,0,0); - MPD_NEW_STATIC(t2, 0,0,0,0); - MPD_NEW_STATIC(ulp, 0,0,0,0); - MPD_NEW_STATIC(aa, 0,0,0,0); - mpd_ssize_t prec; - - if (result == a) { - if (!mpd_qcopy(&aa, a, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - a = &aa; - } - - workctx.clamp = 0; - prec = ctx->prec + 3; - while (1) { - workctx.prec = prec; - _mpd_qln(result, a, &workctx, status); - _ssettriple(&ulp, MPD_POS, 1, - result->exp + result->digits-workctx.prec); - - workctx.prec = ctx->prec; - mpd_qadd(&t1, result, &ulp, &workctx, &workctx.status); - mpd_qsub(&t2, result, &ulp, &workctx, &workctx.status); - if (mpd_isspecial(result) || mpd_iszerocoeff(result) || - mpd_qcmp(&t1, &t2, status) == 0) { - workctx.clamp = ctx->clamp; - mpd_check_underflow(result, &workctx, status); - mpd_qfinalize(result, &workctx, status); - break; - } - prec += MPD_RDIGITS; - } - mpd_del(&t1); - mpd_del(&t2); - mpd_del(&ulp); - mpd_del(&aa); - } - else { - _mpd_qln(result, a, &workctx, status); - mpd_check_underflow(result, &workctx, status); - mpd_qfinalize(result, &workctx, status); - } -} - -/* - * Internal log10() function that does not check for specials, zero or one. - * Case SKIP_FINALIZE: - * Relative error: abs(result - log10(a)) < 0.1 * 10**-prec * abs(log10(a)) - * Case DO_FINALIZE: - * Ulp error: abs(result - log10(a)) < ulp(log10(a)) - */ -enum {SKIP_FINALIZE, DO_FINALIZE}; -static void -_mpd_qlog10(int action, mpd_t *result, const mpd_t *a, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t workctx; - MPD_NEW_STATIC(ln10,0,0,0,0); - - mpd_maxcontext(&workctx); - workctx.prec = ctx->prec + 3; - /* relative error: 0.1 * 10**(-p-3). The specific underflow shortcut - * in _mpd_qln() does not change the final result. */ - _mpd_qln(result, a, &workctx, status); - /* relative error: 5 * 10**(-p-3) */ - mpd_qln10(&ln10, workctx.prec, status); - - if (action == DO_FINALIZE) { - workctx = *ctx; - workctx.round = MPD_ROUND_HALF_EVEN; - } - /* SKIP_FINALIZE: relative error: 5 * 10**(-p-3) */ - _mpd_qdiv(NO_IDEAL_EXP, result, result, &ln10, &workctx, status); - - mpd_del(&ln10); -} - -/* log10(a) */ -void -mpd_qlog10(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx; - mpd_ssize_t adjexp, t; - - workctx = *ctx; - workctx.round = MPD_ROUND_HALF_EVEN; - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - if (mpd_isnegative(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - mpd_setspecial(result, MPD_POS, MPD_INF); - return; - } - if (mpd_iszerocoeff(a)) { - mpd_setspecial(result, MPD_NEG, MPD_INF); - return; - } - if (mpd_isnegative(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_coeff_ispow10(a)) { - uint8_t sign = 0; - adjexp = mpd_adjexp(a); - if (adjexp < 0) { - sign = 1; - adjexp = -adjexp; - } - _settriple(result, sign, adjexp, 0); - mpd_qfinalize(result, &workctx, status); - return; - } - /* - * Check if the result will overflow (0 < x, x != 1): - * 1) log10(x) < 0 iff adjexp(x) < 0 - * 2) 0 < x /\ x <= y ==> adjexp(x) <= adjexp(y) - * 3) adjexp(x) <= log10(x) < adjexp(x) + 1 - * - * Case adjexp(x) >= 0: - * 4) adjexp(x) <= abs(log10(x)) - * Case adjexp(x) > 0: - * 5) adjexp(adjexp(x)) <= adjexp(abs(log10(x))) - * Case adjexp(x) == 0: - * mpd_exp_digits(t)-1 == 0 <= emax (the shortcut is not triggered) - * - * Case adjexp(x) < 0: - * 6) -adjexp(x) - 1 < abs(log10(x)) - * Case adjexp(x) < -1: - * 7) adjexp(-adjexp(x) - 1) <= adjexp(abs(log(x))) - * Case adjexp(x) == -1: - * mpd_exp_digits(t)-1 == 0 <= emax (the shortcut is not triggered) - */ - adjexp = mpd_adjexp(a); - t = (adjexp < 0) ? -adjexp-1 : adjexp; - if (mpd_exp_digits(t)-1 > ctx->emax) { - *status |= MPD_Overflow|MPD_Inexact|MPD_Rounded; - mpd_setspecial(result, (adjexp<0), MPD_INF); - return; - } - - if (ctx->allcr) { - MPD_NEW_STATIC(t1, 0,0,0,0); - MPD_NEW_STATIC(t2, 0,0,0,0); - MPD_NEW_STATIC(ulp, 0,0,0,0); - MPD_NEW_STATIC(aa, 0,0,0,0); - mpd_ssize_t prec; - - if (result == a) { - if (!mpd_qcopy(&aa, a, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - a = &aa; - } - - workctx.clamp = 0; - prec = ctx->prec + 3; - while (1) { - workctx.prec = prec; - _mpd_qlog10(SKIP_FINALIZE, result, a, &workctx, status); - _ssettriple(&ulp, MPD_POS, 1, - result->exp + result->digits-workctx.prec); - - workctx.prec = ctx->prec; - mpd_qadd(&t1, result, &ulp, &workctx, &workctx.status); - mpd_qsub(&t2, result, &ulp, &workctx, &workctx.status); - if (mpd_isspecial(result) || mpd_iszerocoeff(result) || - mpd_qcmp(&t1, &t2, status) == 0) { - workctx.clamp = ctx->clamp; - mpd_check_underflow(result, &workctx, status); - mpd_qfinalize(result, &workctx, status); - break; - } - prec += MPD_RDIGITS; - } - mpd_del(&t1); - mpd_del(&t2); - mpd_del(&ulp); - mpd_del(&aa); - } - else { - _mpd_qlog10(DO_FINALIZE, result, a, &workctx, status); - mpd_check_underflow(result, &workctx, status); - } -} - -/* - * Maximum of the two operands. Attention: If one operand is a quiet NaN and the - * other is numeric, the numeric operand is returned. This may not be what one - * expects. - */ -void -mpd_qmax(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - int c; - - if (mpd_isqnan(a) && !mpd_isnan(b)) { - mpd_qcopy(result, b, status); - } - else if (mpd_isqnan(b) && !mpd_isnan(a)) { - mpd_qcopy(result, a, status); - } - else if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - else { - c = _mpd_cmp(a, b); - if (c == 0) { - c = _mpd_cmp_numequal(a, b); - } - - if (c < 0) { - mpd_qcopy(result, b, status); - } - else { - mpd_qcopy(result, a, status); - } - } - - mpd_qfinalize(result, ctx, status); -} - -/* - * Maximum magnitude: Same as mpd_max(), but compares the operands with their - * sign ignored. - */ -void -mpd_qmax_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - int c; - - if (mpd_isqnan(a) && !mpd_isnan(b)) { - mpd_qcopy(result, b, status); - } - else if (mpd_isqnan(b) && !mpd_isnan(a)) { - mpd_qcopy(result, a, status); - } - else if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - else { - c = _mpd_cmp_abs(a, b); - if (c == 0) { - c = _mpd_cmp_numequal(a, b); - } - - if (c < 0) { - mpd_qcopy(result, b, status); - } - else { - mpd_qcopy(result, a, status); - } - } - - mpd_qfinalize(result, ctx, status); -} - -/* - * Minimum of the two operands. Attention: If one operand is a quiet NaN and the - * other is numeric, the numeric operand is returned. This may not be what one - * expects. - */ -void -mpd_qmin(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - int c; - - if (mpd_isqnan(a) && !mpd_isnan(b)) { - mpd_qcopy(result, b, status); - } - else if (mpd_isqnan(b) && !mpd_isnan(a)) { - mpd_qcopy(result, a, status); - } - else if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - else { - c = _mpd_cmp(a, b); - if (c == 0) { - c = _mpd_cmp_numequal(a, b); - } - - if (c < 0) { - mpd_qcopy(result, a, status); - } - else { - mpd_qcopy(result, b, status); - } - } - - mpd_qfinalize(result, ctx, status); -} - -/* - * Minimum magnitude: Same as mpd_min(), but compares the operands with their - * sign ignored. - */ -void -mpd_qmin_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - int c; - - if (mpd_isqnan(a) && !mpd_isnan(b)) { - mpd_qcopy(result, b, status); - } - else if (mpd_isqnan(b) && !mpd_isnan(a)) { - mpd_qcopy(result, a, status); - } - else if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - else { - c = _mpd_cmp_abs(a, b); - if (c == 0) { - c = _mpd_cmp_numequal(a, b); - } - - if (c < 0) { - mpd_qcopy(result, a, status); - } - else { - mpd_qcopy(result, b, status); - } - } - - mpd_qfinalize(result, ctx, status); -} - -/* Minimum space needed for the result array in _karatsuba_rec(). */ -static inline mpd_size_t -_kmul_resultsize(mpd_size_t la, mpd_size_t lb) -{ - mpd_size_t n, m; - - n = add_size_t(la, lb); - n = add_size_t(n, 1); - - m = (la+1)/2 + 1; - m = mul_size_t(m, 3); - - return (m > n) ? m : n; -} - -/* Work space needed in _karatsuba_rec(). lim >= 4 */ -static inline mpd_size_t -_kmul_worksize(mpd_size_t n, mpd_size_t lim) -{ - mpd_size_t m; - - if (n <= lim) { - return 0; - } - - m = (n+1)/2 + 1; - - return add_size_t(mul_size_t(m, 2), _kmul_worksize(m, lim)); -} - - -#define MPD_KARATSUBA_BASECASE 16 /* must be >= 4 */ - -/* - * Add the product of a and b to c. - * c must be _kmul_resultsize(la, lb) in size. - * w is used as a work array and must be _kmul_worksize(a, lim) in size. - * Roman E. Maeder, Storage Allocation for the Karatsuba Integer Multiplication - * Algorithm. In "Design and implementation of symbolic computation systems", - * Springer, 1993, ISBN 354057235X, 9783540572350. - */ -static void -_karatsuba_rec(mpd_uint_t *c, const mpd_uint_t *a, const mpd_uint_t *b, - mpd_uint_t *w, mpd_size_t la, mpd_size_t lb) -{ - mpd_size_t m, lt; - - assert(la >= lb && lb > 0); - assert(la <= MPD_KARATSUBA_BASECASE || w != NULL); - - if (la <= MPD_KARATSUBA_BASECASE) { - _mpd_basemul(c, a, b, la, lb); - return; - } - - m = (la+1)/2; /* ceil(la/2) */ - - /* lb <= m < la */ - if (lb <= m) { - - /* lb can now be larger than la-m */ - if (lb > la-m) { - lt = lb + lb + 1; /* space needed for result array */ - mpd_uint_zero(w, lt); /* clear result array */ - _karatsuba_rec(w, b, a+m, w+lt, lb, la-m); /* b*ah */ - } - else { - lt = (la-m) + (la-m) + 1; /* space needed for result array */ - mpd_uint_zero(w, lt); /* clear result array */ - _karatsuba_rec(w, a+m, b, w+lt, la-m, lb); /* ah*b */ - } - _mpd_baseaddto(c+m, w, (la-m)+lb); /* add ah*b*B**m */ - - lt = m + m + 1; /* space needed for the result array */ - mpd_uint_zero(w, lt); /* clear result array */ - _karatsuba_rec(w, a, b, w+lt, m, lb); /* al*b */ - _mpd_baseaddto(c, w, m+lb); /* add al*b */ - - return; - } - - /* la >= lb > m */ - memcpy(w, a, m * sizeof *w); - w[m] = 0; - _mpd_baseaddto(w, a+m, la-m); - - memcpy(w+(m+1), b, m * sizeof *w); - w[m+1+m] = 0; - _mpd_baseaddto(w+(m+1), b+m, lb-m); - - _karatsuba_rec(c+m, w, w+(m+1), w+2*(m+1), m+1, m+1); - - lt = (la-m) + (la-m) + 1; - mpd_uint_zero(w, lt); - - _karatsuba_rec(w, a+m, b+m, w+lt, la-m, lb-m); - - _mpd_baseaddto(c+2*m, w, (la-m) + (lb-m)); - _mpd_basesubfrom(c+m, w, (la-m) + (lb-m)); - - lt = m + m + 1; - mpd_uint_zero(w, lt); - - _karatsuba_rec(w, a, b, w+lt, m, m); - _mpd_baseaddto(c, w, m+m); - _mpd_basesubfrom(c+m, w, m+m); - - return; -} - -/* - * Multiply u and v, using Karatsuba multiplication. Returns a pointer - * to the result or NULL in case of failure (malloc error). - * Conditions: ulen >= vlen, ulen >= 4 - */ -static mpd_uint_t * -_mpd_kmul(const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t ulen, mpd_size_t vlen, - mpd_size_t *rsize) -{ - mpd_uint_t *result = NULL, *w = NULL; - mpd_size_t m; - - assert(ulen >= 4); - assert(ulen >= vlen); - - *rsize = _kmul_resultsize(ulen, vlen); - if ((result = mpd_calloc(*rsize, sizeof *result)) == NULL) { - return NULL; - } - - m = _kmul_worksize(ulen, MPD_KARATSUBA_BASECASE); - if (m && ((w = mpd_calloc(m, sizeof *w)) == NULL)) { - mpd_free(result); - return NULL; - } - - _karatsuba_rec(result, u, v, w, ulen, vlen); - - - if (w) mpd_free(w); - return result; -} - - -/* - * Determine the minimum length for the number theoretic transform. Valid - * transform lengths are 2**n or 3*2**n, where 2**n <= MPD_MAXTRANSFORM_2N. - * The function finds the shortest length m such that rsize <= m. - */ -static inline mpd_size_t -_mpd_get_transform_len(mpd_size_t rsize) -{ - mpd_size_t log2rsize; - mpd_size_t x, step; - - assert(rsize >= 4); - log2rsize = mpd_bsr(rsize); - - if (rsize <= 1024) { - /* 2**n is faster in this range. */ - x = ((mpd_size_t)1)<<log2rsize; - return (rsize == x) ? x : x<<1; - } - else if (rsize <= MPD_MAXTRANSFORM_2N) { - x = ((mpd_size_t)1)<<log2rsize; - if (rsize == x) return x; - step = x>>1; - x += step; - return (rsize <= x) ? x : x + step; - } - else if (rsize <= MPD_MAXTRANSFORM_2N+MPD_MAXTRANSFORM_2N/2) { - return MPD_MAXTRANSFORM_2N+MPD_MAXTRANSFORM_2N/2; - } - else if (rsize <= 3*MPD_MAXTRANSFORM_2N) { - return 3*MPD_MAXTRANSFORM_2N; - } - else { - return MPD_SIZE_MAX; - } -} - -#ifdef PPRO -#ifndef _MSC_VER -static inline unsigned short -_mpd_get_control87(void) -{ - unsigned short cw; - - __asm__ __volatile__ ("fnstcw %0" : "=m" (cw)); - return cw; -} - -static inline void -_mpd_set_control87(unsigned short cw) -{ - __asm__ __volatile__ ("fldcw %0" : : "m" (cw)); -} -#endif - -static unsigned int -mpd_set_fenv(void) -{ - unsigned int cw; -#ifdef _MSC_VER - unsigned int flags = - _EM_INVALID|_EM_DENORMAL|_EM_ZERODIVIDE|_EM_OVERFLOW| - _EM_UNDERFLOW|_EM_INEXACT|_RC_CHOP|_PC_64; - unsigned int mask = _MCW_EM|_MCW_RC|_MCW_PC; - unsigned int dummy; - - __control87_2(0, 0, &cw, NULL); - __control87_2(flags, mask, &dummy, NULL); -#else - cw = _mpd_get_control87(); - _mpd_set_control87(cw|0xF3F); -#endif - return cw; -} - -static void -mpd_restore_fenv(unsigned int cw) -{ -#ifdef _MSC_VER - unsigned int mask = _MCW_EM|_MCW_RC|_MCW_PC; - unsigned int dummy; - - __control87_2(cw, mask, &dummy, NULL); -#else - _mpd_set_control87((unsigned short)cw); -#endif -} -#endif /* PPRO */ - -/* - * Multiply u and v, using the fast number theoretic transform. Returns - * a pointer to the result or NULL in case of failure (malloc error). - */ -static mpd_uint_t * -_mpd_fntmul(const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t ulen, mpd_size_t vlen, - mpd_size_t *rsize) -{ - mpd_uint_t *c1 = NULL, *c2 = NULL, *c3 = NULL, *vtmp = NULL; - mpd_size_t n; - -#ifdef PPRO - unsigned int cw; - cw = mpd_set_fenv(); -#endif - - *rsize = add_size_t(ulen, vlen); - if ((n = _mpd_get_transform_len(*rsize)) == MPD_SIZE_MAX) { - goto malloc_error; - } - - if ((c1 = mpd_calloc(n, sizeof *c1)) == NULL) { - goto malloc_error; - } - if ((c2 = mpd_calloc(n, sizeof *c2)) == NULL) { - goto malloc_error; - } - if ((c3 = mpd_calloc(n, sizeof *c3)) == NULL) { - goto malloc_error; - } - - memcpy(c1, u, ulen * (sizeof *c1)); - memcpy(c2, u, ulen * (sizeof *c2)); - memcpy(c3, u, ulen * (sizeof *c3)); - - if (u == v) { - if (!fnt_autoconvolute(c1, n, P1) || - !fnt_autoconvolute(c2, n, P2) || - !fnt_autoconvolute(c3, n, P3)) { - goto malloc_error; - } - } - else { - if ((vtmp = mpd_calloc(n, sizeof *vtmp)) == NULL) { - goto malloc_error; - } - - memcpy(vtmp, v, vlen * (sizeof *vtmp)); - if (!fnt_convolute(c1, vtmp, n, P1)) { - mpd_free(vtmp); - goto malloc_error; - } - - memcpy(vtmp, v, vlen * (sizeof *vtmp)); - mpd_uint_zero(vtmp+vlen, n-vlen); - if (!fnt_convolute(c2, vtmp, n, P2)) { - mpd_free(vtmp); - goto malloc_error; - } - - memcpy(vtmp, v, vlen * (sizeof *vtmp)); - mpd_uint_zero(vtmp+vlen, n-vlen); - if (!fnt_convolute(c3, vtmp, n, P3)) { - mpd_free(vtmp); - goto malloc_error; - } - - mpd_free(vtmp); - } - - crt3(c1, c2, c3, *rsize); - -out: -#ifdef PPRO - mpd_restore_fenv(cw); -#endif - if (c2) mpd_free(c2); - if (c3) mpd_free(c3); - return c1; - -malloc_error: - if (c1) mpd_free(c1); - c1 = NULL; - goto out; -} - - -/* - * Karatsuba multiplication with FNT/basemul as the base case. - */ -static int -_karatsuba_rec_fnt(mpd_uint_t *c, const mpd_uint_t *a, const mpd_uint_t *b, - mpd_uint_t *w, mpd_size_t la, mpd_size_t lb) -{ - mpd_size_t m, lt; - - assert(la >= lb && lb > 0); - assert(la <= 3*(MPD_MAXTRANSFORM_2N/2) || w != NULL); - - if (la <= 3*(MPD_MAXTRANSFORM_2N/2)) { - - if (lb <= 192) { - _mpd_basemul(c, b, a, lb, la); - } - else { - mpd_uint_t *result; - mpd_size_t dummy; - - if ((result = _mpd_fntmul(a, b, la, lb, &dummy)) == NULL) { - return 0; - } - memcpy(c, result, (la+lb) * (sizeof *result)); - mpd_free(result); - } - return 1; - } - - m = (la+1)/2; /* ceil(la/2) */ - - /* lb <= m < la */ - if (lb <= m) { - - /* lb can now be larger than la-m */ - if (lb > la-m) { - lt = lb + lb + 1; /* space needed for result array */ - mpd_uint_zero(w, lt); /* clear result array */ - if (!_karatsuba_rec_fnt(w, b, a+m, w+lt, lb, la-m)) { /* b*ah */ - return 0; /* GCOV_UNLIKELY */ - } - } - else { - lt = (la-m) + (la-m) + 1; /* space needed for result array */ - mpd_uint_zero(w, lt); /* clear result array */ - if (!_karatsuba_rec_fnt(w, a+m, b, w+lt, la-m, lb)) { /* ah*b */ - return 0; /* GCOV_UNLIKELY */ - } - } - _mpd_baseaddto(c+m, w, (la-m)+lb); /* add ah*b*B**m */ - - lt = m + m + 1; /* space needed for the result array */ - mpd_uint_zero(w, lt); /* clear result array */ - if (!_karatsuba_rec_fnt(w, a, b, w+lt, m, lb)) { /* al*b */ - return 0; /* GCOV_UNLIKELY */ - } - _mpd_baseaddto(c, w, m+lb); /* add al*b */ - - return 1; - } - - /* la >= lb > m */ - memcpy(w, a, m * sizeof *w); - w[m] = 0; - _mpd_baseaddto(w, a+m, la-m); - - memcpy(w+(m+1), b, m * sizeof *w); - w[m+1+m] = 0; - _mpd_baseaddto(w+(m+1), b+m, lb-m); - - if (!_karatsuba_rec_fnt(c+m, w, w+(m+1), w+2*(m+1), m+1, m+1)) { - return 0; /* GCOV_UNLIKELY */ - } - - lt = (la-m) + (la-m) + 1; - mpd_uint_zero(w, lt); - - if (!_karatsuba_rec_fnt(w, a+m, b+m, w+lt, la-m, lb-m)) { - return 0; /* GCOV_UNLIKELY */ - } - - _mpd_baseaddto(c+2*m, w, (la-m) + (lb-m)); - _mpd_basesubfrom(c+m, w, (la-m) + (lb-m)); - - lt = m + m + 1; - mpd_uint_zero(w, lt); - - if (!_karatsuba_rec_fnt(w, a, b, w+lt, m, m)) { - return 0; /* GCOV_UNLIKELY */ - } - _mpd_baseaddto(c, w, m+m); - _mpd_basesubfrom(c+m, w, m+m); - - return 1; -} - -/* - * Multiply u and v, using Karatsuba multiplication with the FNT as the - * base case. Returns a pointer to the result or NULL in case of failure - * (malloc error). Conditions: ulen >= vlen, ulen >= 4. - */ -static mpd_uint_t * -_mpd_kmul_fnt(const mpd_uint_t *u, const mpd_uint_t *v, - mpd_size_t ulen, mpd_size_t vlen, - mpd_size_t *rsize) -{ - mpd_uint_t *result = NULL, *w = NULL; - mpd_size_t m; - - assert(ulen >= 4); - assert(ulen >= vlen); - - *rsize = _kmul_resultsize(ulen, vlen); - if ((result = mpd_calloc(*rsize, sizeof *result)) == NULL) { - return NULL; - } - - m = _kmul_worksize(ulen, 3*(MPD_MAXTRANSFORM_2N/2)); - if (m && ((w = mpd_calloc(m, sizeof *w)) == NULL)) { - mpd_free(result); /* GCOV_UNLIKELY */ - return NULL; /* GCOV_UNLIKELY */ - } - - if (!_karatsuba_rec_fnt(result, u, v, w, ulen, vlen)) { - mpd_free(result); - result = NULL; - } - - - if (w) mpd_free(w); - return result; -} - - -/* Deal with the special cases of multiplying infinities. */ -static void -_mpd_qmul_inf(mpd_t *result, const mpd_t *a, const mpd_t *b, uint32_t *status) -{ - if (mpd_isinfinite(a)) { - if (mpd_iszero(b)) { - mpd_seterror(result, MPD_Invalid_operation, status); - } - else { - mpd_setspecial(result, mpd_sign(a)^mpd_sign(b), MPD_INF); - } - return; - } - assert(mpd_isinfinite(b)); - if (mpd_iszero(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - } - else { - mpd_setspecial(result, mpd_sign(a)^mpd_sign(b), MPD_INF); - } -} - -/* - * Internal function: Multiply a and b. _mpd_qmul deals with specials but - * does NOT finalize the result. This is for use in mpd_fma(). - */ -static inline void -_mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - const mpd_t *big = a, *small = b; - mpd_uint_t *rdata = NULL; - mpd_uint_t rbuf[MPD_MINALLOC_MAX]; - mpd_size_t rsize, i; - - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - _mpd_qmul_inf(result, a, b, status); - return; - } - - if (small->len > big->len) { - _mpd_ptrswap(&big, &small); - } - - rsize = big->len + small->len; - - if (big->len == 1) { - _mpd_singlemul(result->data, big->data[0], small->data[0]); - goto finish; - } - if (rsize <= (mpd_size_t)MPD_MINALLOC_MAX) { - if (big->len == 2) { - _mpd_mul_2_le2(rbuf, big->data, small->data, small->len); - } - else { - mpd_uint_zero(rbuf, rsize); - if (small->len == 1) { - _mpd_shortmul(rbuf, big->data, big->len, small->data[0]); - } - else { - _mpd_basemul(rbuf, small->data, big->data, small->len, big->len); - } - } - if (!mpd_qresize(result, rsize, status)) { - return; - } - for(i = 0; i < rsize; i++) { - result->data[i] = rbuf[i]; - } - goto finish; - } - - - if (small->len <= 256) { - rdata = mpd_calloc(rsize, sizeof *rdata); - if (rdata != NULL) { - if (small->len == 1) { - _mpd_shortmul(rdata, big->data, big->len, small->data[0]); - } - else { - _mpd_basemul(rdata, small->data, big->data, small->len, big->len); - } - } - } - else if (rsize <= 1024) { - rdata = _mpd_kmul(big->data, small->data, big->len, small->len, &rsize); - } - else if (rsize <= 3*MPD_MAXTRANSFORM_2N) { - rdata = _mpd_fntmul(big->data, small->data, big->len, small->len, &rsize); - } - else { - rdata = _mpd_kmul_fnt(big->data, small->data, big->len, small->len, &rsize); - } - - if (rdata == NULL) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - - if (mpd_isdynamic_data(result)) { - mpd_free(result->data); - } - result->data = rdata; - result->alloc = rsize; - mpd_set_dynamic_data(result); - - -finish: - mpd_set_flags(result, mpd_sign(a)^mpd_sign(b)); - result->exp = big->exp + small->exp; - result->len = _mpd_real_size(result->data, rsize); - /* resize to smaller cannot fail */ - mpd_qresize(result, result->len, status); - mpd_setdigits(result); -} - -/* Multiply a and b. */ -void -mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - _mpd_qmul(result, a, b, ctx, status); - mpd_qfinalize(result, ctx, status); -} - -/* Multiply a and b. Set NaN/Invalid_operation if the result is inexact. */ -static void -_mpd_qmul_exact(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - - mpd_qmul(result, a, b, ctx, &workstatus); - *status |= workstatus; - if (workstatus & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { - mpd_seterror(result, MPD_Invalid_operation, status); - } -} - -/* Multiply decimal and mpd_ssize_t. */ -void -mpd_qmul_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qsset_ssize(&bb, b, &maxcontext, status); - mpd_qmul(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Multiply decimal and mpd_uint_t. */ -void -mpd_qmul_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qsset_uint(&bb, b, &maxcontext, status); - mpd_qmul(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -void -mpd_qmul_i32(mpd_t *result, const mpd_t *a, int32_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qmul_ssize(result, a, b, ctx, status); -} - -void -mpd_qmul_u32(mpd_t *result, const mpd_t *a, uint32_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qmul_uint(result, a, b, ctx, status); -} - -#ifdef CONFIG_64 -void -mpd_qmul_i64(mpd_t *result, const mpd_t *a, int64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qmul_ssize(result, a, b, ctx, status); -} - -void -mpd_qmul_u64(mpd_t *result, const mpd_t *a, uint64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_qmul_uint(result, a, b, ctx, status); -} -#elif !defined(LEGACY_COMPILER) -/* Multiply decimal and int64_t. */ -void -mpd_qmul_i64(mpd_t *result, const mpd_t *a, int64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qset_i64(&bb, b, &maxcontext, status); - mpd_qmul(result, a, &bb, ctx, status); - mpd_del(&bb); -} - -/* Multiply decimal and uint64_t. */ -void -mpd_qmul_u64(mpd_t *result, const mpd_t *a, uint64_t b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(bb,0,0,0,0); - - mpd_maxcontext(&maxcontext); - mpd_qset_u64(&bb, b, &maxcontext, status); - mpd_qmul(result, a, &bb, ctx, status); - mpd_del(&bb); -} -#endif - -/* Like the minus operator. */ -void -mpd_qminus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - } - - if (mpd_iszero(a) && ctx->round != MPD_ROUND_FLOOR) { - mpd_qcopy_abs(result, a, status); - } - else { - mpd_qcopy_negate(result, a, status); - } - - mpd_qfinalize(result, ctx, status); -} - -/* Like the plus operator. */ -void -mpd_qplus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - } - - if (mpd_iszero(a) && ctx->round != MPD_ROUND_FLOOR) { - mpd_qcopy_abs(result, a, status); - } - else { - mpd_qcopy(result, a, status); - } - - mpd_qfinalize(result, ctx, status); -} - -/* The largest representable number that is smaller than the operand. */ -void -mpd_qnext_minus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx; - MPD_NEW_CONST(tiny,MPD_POS,mpd_etiny(ctx)-1,1,1,1,1); - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - - assert(mpd_isinfinite(a)); - if (mpd_isnegative(a)) { - mpd_qcopy(result, a, status); - return; - } - else { - mpd_clear_flags(result); - mpd_qmaxcoeff(result, ctx, status); - if (mpd_isnan(result)) { - return; - } - result->exp = mpd_etop(ctx); - return; - } - } - - mpd_workcontext(&workctx, ctx); - workctx.round = MPD_ROUND_FLOOR; - - if (!mpd_qcopy(result, a, status)) { - return; - } - - mpd_qfinalize(result, &workctx, &workctx.status); - if (workctx.status&(MPD_Inexact|MPD_Errors)) { - *status |= (workctx.status&MPD_Errors); - return; - } - - workctx.status = 0; - mpd_qsub(result, a, &tiny, &workctx, &workctx.status); - *status |= (workctx.status&MPD_Errors); -} - -/* The smallest representable number that is larger than the operand. */ -void -mpd_qnext_plus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx; - MPD_NEW_CONST(tiny,MPD_POS,mpd_etiny(ctx)-1,1,1,1,1); - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - - assert(mpd_isinfinite(a)); - if (mpd_ispositive(a)) { - mpd_qcopy(result, a, status); - } - else { - mpd_clear_flags(result); - mpd_qmaxcoeff(result, ctx, status); - if (mpd_isnan(result)) { - return; - } - mpd_set_flags(result, MPD_NEG); - result->exp = mpd_etop(ctx); - } - return; - } - - mpd_workcontext(&workctx, ctx); - workctx.round = MPD_ROUND_CEILING; - - if (!mpd_qcopy(result, a, status)) { - return; - } - - mpd_qfinalize(result, &workctx, &workctx.status); - if (workctx.status & (MPD_Inexact|MPD_Errors)) { - *status |= (workctx.status&MPD_Errors); - return; - } - - workctx.status = 0; - mpd_qadd(result, a, &tiny, &workctx, &workctx.status); - *status |= (workctx.status&MPD_Errors); -} - -/* - * The number closest to the first operand that is in the direction towards - * the second operand. - */ -void -mpd_qnext_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - int c; - - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - - c = _mpd_cmp(a, b); - if (c == 0) { - mpd_qcopy_sign(result, a, b, status); - return; - } - - if (c < 0) { - mpd_qnext_plus(result, a, ctx, status); - } - else { - mpd_qnext_minus(result, a, ctx, status); - } - - if (mpd_isinfinite(result)) { - *status |= (MPD_Overflow|MPD_Rounded|MPD_Inexact); - } - else if (mpd_adjexp(result) < ctx->emin) { - *status |= (MPD_Underflow|MPD_Subnormal|MPD_Rounded|MPD_Inexact); - if (mpd_iszero(result)) { - *status |= MPD_Clamped; - } - } -} - -/* - * Internal function: Integer power with mpd_uint_t exponent. The function - * can fail with MPD_Malloc_error. - * - * The error is equal to the error incurred in k-1 multiplications. Assuming - * the upper bound for the relative error in each operation: - * - * abs(err) = 5 * 10**-prec - * result = x**k * (1 + err)**(k-1) - */ -static inline void -_mpd_qpow_uint(mpd_t *result, const mpd_t *base, mpd_uint_t exp, - uint8_t resultsign, const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_uint_t n; - - if (exp == 0) { - _settriple(result, resultsign, 1, 0); /* GCOV_NOT_REACHED */ - return; /* GCOV_NOT_REACHED */ - } - - if (!mpd_qcopy(result, base, status)) { - return; - } - - n = mpd_bits[mpd_bsr(exp)]; - while (n >>= 1) { - mpd_qmul(result, result, result, ctx, &workstatus); - if (exp & n) { - mpd_qmul(result, result, base, ctx, &workstatus); - } - if (mpd_isspecial(result) || - (mpd_iszerocoeff(result) && (workstatus & MPD_Clamped))) { - break; - } - } - - *status |= workstatus; - mpd_set_sign(result, resultsign); -} - -/* - * Internal function: Integer power with mpd_t exponent, tbase and texp - * are modified!! Function can fail with MPD_Malloc_error. - * - * The error is equal to the error incurred in k multiplications. Assuming - * the upper bound for the relative error in each operation: - * - * abs(err) = 5 * 10**-prec - * result = x**k * (1 + err)**k - */ -static inline void -_mpd_qpow_mpd(mpd_t *result, mpd_t *tbase, mpd_t *texp, uint8_t resultsign, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_context_t maxctx; - MPD_NEW_CONST(two,0,0,1,1,1,2); - - - mpd_maxcontext(&maxctx); - - /* resize to smaller cannot fail */ - mpd_qcopy(result, &one, status); - - while (!mpd_iszero(texp)) { - if (mpd_isodd(texp)) { - mpd_qmul(result, result, tbase, ctx, &workstatus); - *status |= workstatus; - if (mpd_isspecial(result) || - (mpd_iszerocoeff(result) && (workstatus & MPD_Clamped))) { - break; - } - } - mpd_qmul(tbase, tbase, tbase, ctx, &workstatus); - mpd_qdivint(texp, texp, &two, &maxctx, &workstatus); - if (mpd_isnan(tbase) || mpd_isnan(texp)) { - mpd_seterror(result, workstatus&MPD_Errors, status); - return; - } - } - mpd_set_sign(result, resultsign); -} - -/* - * The power function for integer exponents. Relative error _before_ the - * final rounding to prec: - * abs(result - base**exp) < 0.1 * 10**-prec * abs(base**exp) - */ -static void -_mpd_qpow_int(mpd_t *result, const mpd_t *base, const mpd_t *exp, - uint8_t resultsign, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t workctx; - MPD_NEW_STATIC(tbase,0,0,0,0); - MPD_NEW_STATIC(texp,0,0,0,0); - mpd_uint_t n; - - - mpd_workcontext(&workctx, ctx); - workctx.prec += (exp->digits + exp->exp + 2); - workctx.round = MPD_ROUND_HALF_EVEN; - workctx.clamp = 0; - if (mpd_isnegative(exp)) { - uint32_t workstatus = 0; - workctx.prec += 1; - mpd_qdiv(&tbase, &one, base, &workctx, &workstatus); - *status |= workstatus; - if (workstatus&MPD_Errors) { - mpd_setspecial(result, MPD_POS, MPD_NAN); - goto finish; - } - } - else { - if (!mpd_qcopy(&tbase, base, status)) { - mpd_setspecial(result, MPD_POS, MPD_NAN); - goto finish; - } - } - - n = mpd_qabs_uint(exp, &workctx.status); - if (workctx.status&MPD_Invalid_operation) { - if (!mpd_qcopy(&texp, exp, status)) { - mpd_setspecial(result, MPD_POS, MPD_NAN); /* GCOV_UNLIKELY */ - goto finish; /* GCOV_UNLIKELY */ - } - _mpd_qpow_mpd(result, &tbase, &texp, resultsign, &workctx, status); - } - else { - _mpd_qpow_uint(result, &tbase, n, resultsign, &workctx, status); - } - - if (mpd_isinfinite(result)) { - /* for ROUND_DOWN, ROUND_FLOOR, etc. */ - _settriple(result, resultsign, 1, MPD_EXP_INF); - } - -finish: - mpd_del(&tbase); - mpd_del(&texp); - mpd_qfinalize(result, ctx, status); -} - -/* - * If the exponent is infinite and base equals one, the result is one - * with a coefficient of length prec. Otherwise, result is undefined. - * Return the value of the comparison against one. - */ -static int -_qcheck_pow_one_inf(mpd_t *result, const mpd_t *base, uint8_t resultsign, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t shift; - int cmp; - - if ((cmp = _mpd_cmp(base, &one)) == 0) { - shift = ctx->prec-1; - mpd_qshiftl(result, &one, shift, status); - result->exp = -shift; - mpd_set_flags(result, resultsign); - *status |= (MPD_Inexact|MPD_Rounded); - } - - return cmp; -} - -/* - * If abs(base) equals one, calculate the correct power of one result. - * Otherwise, result is undefined. Return the value of the comparison - * against 1. - * - * This is an internal function that does not check for specials. - */ -static int -_qcheck_pow_one(mpd_t *result, const mpd_t *base, const mpd_t *exp, - uint8_t resultsign, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_ssize_t shift; - int cmp; - - if ((cmp = _mpd_cmp_abs(base, &one)) == 0) { - if (_mpd_isint(exp)) { - if (mpd_isnegative(exp)) { - _settriple(result, resultsign, 1, 0); - return 0; - } - /* 1.000**3 = 1.000000000 */ - mpd_qmul_ssize(result, exp, -base->exp, ctx, &workstatus); - if (workstatus&MPD_Errors) { - *status |= (workstatus&MPD_Errors); - return 0; - } - /* digits-1 after exponentiation */ - shift = mpd_qget_ssize(result, &workstatus); - /* shift is MPD_SSIZE_MAX if result is too large */ - if (shift > ctx->prec-1) { - shift = ctx->prec-1; - *status |= MPD_Rounded; - } - } - else if (mpd_ispositive(base)) { - shift = ctx->prec-1; - *status |= (MPD_Inexact|MPD_Rounded); - } - else { - return -2; /* GCOV_NOT_REACHED */ - } - if (!mpd_qshiftl(result, &one, shift, status)) { - return 0; - } - result->exp = -shift; - mpd_set_flags(result, resultsign); - } - - return cmp; -} - -/* - * Detect certain over/underflow of x**y. - * ACL2 proof: pow-bounds.lisp. - * - * Symbols: - * - * e: EXP_INF or EXP_CLAMP - * x: base - * y: exponent - * - * omega(e) = log10(abs(e)) - * zeta(x) = log10(abs(log10(x))) - * theta(y) = log10(abs(y)) - * - * Upper and lower bounds: - * - * ub_omega(e) = ceil(log10(abs(e))) - * lb_theta(y) = floor(log10(abs(y))) - * - * | floor(log10(floor(abs(log10(x))))) if x < 1/10 or x >= 10 - * lb_zeta(x) = | floor(log10(abs(x-1)/10)) if 1/10 <= x < 1 - * | floor(log10(abs((x-1)/100))) if 1 < x < 10 - * - * ub_omega(e) and lb_theta(y) are obviously upper and lower bounds - * for omega(e) and theta(y). - * - * lb_zeta is a lower bound for zeta(x): - * - * x < 1/10 or x >= 10: - * - * abs(log10(x)) >= 1, so the outer log10 is well defined. Since log10 - * is strictly increasing, the end result is a lower bound. - * - * 1/10 <= x < 1: - * - * We use: log10(x) <= (x-1)/log(10) - * abs(log10(x)) >= abs(x-1)/log(10) - * abs(log10(x)) >= abs(x-1)/10 - * - * 1 < x < 10: - * - * We use: (x-1)/(x*log(10)) < log10(x) - * abs((x-1)/100) < abs(log10(x)) - * - * XXX: abs((x-1)/10) would work, need ACL2 proof. - * - * - * Let (0 < x < 1 and y < 0) or (x > 1 and y > 0). (H1) - * Let ub_omega(exp_inf) < lb_zeta(x) + lb_theta(y) (H2) - * - * Then: - * log10(abs(exp_inf)) < log10(abs(log10(x))) + log10(abs(y)). (1) - * exp_inf < log10(x) * y (2) - * 10**exp_inf < x**y (3) - * - * Let (0 < x < 1 and y > 0) or (x > 1 and y < 0). (H3) - * Let ub_omega(exp_clamp) < lb_zeta(x) + lb_theta(y) (H4) - * - * Then: - * log10(abs(exp_clamp)) < log10(abs(log10(x))) + log10(abs(y)). (4) - * log10(x) * y < exp_clamp (5) - * x**y < 10**exp_clamp (6) - * - */ -static mpd_ssize_t -_lower_bound_zeta(const mpd_t *x, uint32_t *status) -{ - mpd_context_t maxctx; - MPD_NEW_STATIC(scratch,0,0,0,0); - mpd_ssize_t t, u; - - t = mpd_adjexp(x); - if (t > 0) { - /* x >= 10 -> floor(log10(floor(abs(log10(x))))) */ - return mpd_exp_digits(t) - 1; - } - else if (t < -1) { - /* x < 1/10 -> floor(log10(floor(abs(log10(x))))) */ - return mpd_exp_digits(t+1) - 1; - } - else { - mpd_maxcontext(&maxctx); - mpd_qsub(&scratch, x, &one, &maxctx, status); - if (mpd_isspecial(&scratch)) { - mpd_del(&scratch); - return MPD_SSIZE_MAX; - } - u = mpd_adjexp(&scratch); - mpd_del(&scratch); - - /* t == -1, 1/10 <= x < 1 -> floor(log10(abs(x-1)/10)) - * t == 0, 1 < x < 10 -> floor(log10(abs(x-1)/100)) */ - return (t == 0) ? u-2 : u-1; - } -} - -/* - * Detect cases of certain overflow/underflow in the power function. - * Assumptions: x != 1, y != 0. The proof above is for positive x. - * If x is negative and y is an odd integer, x**y == -(abs(x)**y), - * so the analysis does not change. - */ -static int -_qcheck_pow_bounds(mpd_t *result, const mpd_t *x, const mpd_t *y, - uint8_t resultsign, - const mpd_context_t *ctx, uint32_t *status) -{ - MPD_NEW_SHARED(abs_x, x); - mpd_ssize_t ub_omega, lb_zeta, lb_theta; - uint8_t sign; - - mpd_set_positive(&abs_x); - - lb_theta = mpd_adjexp(y); - lb_zeta = _lower_bound_zeta(&abs_x, status); - if (lb_zeta == MPD_SSIZE_MAX) { - mpd_seterror(result, MPD_Malloc_error, status); - return 1; - } - - sign = (mpd_adjexp(&abs_x) < 0) ^ mpd_sign(y); - if (sign == 0) { - /* (0 < |x| < 1 and y < 0) or (|x| > 1 and y > 0) */ - ub_omega = mpd_exp_digits(ctx->emax); - if (ub_omega < lb_zeta + lb_theta) { - _settriple(result, resultsign, 1, MPD_EXP_INF); - mpd_qfinalize(result, ctx, status); - return 1; - } - } - else { - /* (0 < |x| < 1 and y > 0) or (|x| > 1 and y < 0). */ - ub_omega = mpd_exp_digits(mpd_etiny(ctx)); - if (ub_omega < lb_zeta + lb_theta) { - _settriple(result, resultsign, 1, mpd_etiny(ctx)-1); - mpd_qfinalize(result, ctx, status); - return 1; - } - } - - return 0; -} - -/* - * TODO: Implement algorithm for computing exact powers from decimal.py. - * In order to prevent infinite loops, this has to be called before - * using Ziv's strategy for correct rounding. - */ -/* -static int -_mpd_qpow_exact(mpd_t *result, const mpd_t *base, const mpd_t *exp, - const mpd_context_t *ctx, uint32_t *status) -{ - return 0; -} -*/ - -/* - * The power function for real exponents. - * Relative error: abs(result - e**y) < e**y * 1/5 * 10**(-prec - 1) - */ -static void -_mpd_qpow_real(mpd_t *result, const mpd_t *base, const mpd_t *exp, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t workctx; - MPD_NEW_STATIC(texp,0,0,0,0); - - if (!mpd_qcopy(&texp, exp, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - - mpd_maxcontext(&workctx); - workctx.prec = (base->digits > ctx->prec) ? base->digits : ctx->prec; - workctx.prec += (4 + MPD_EXPDIGITS); - workctx.round = MPD_ROUND_HALF_EVEN; - workctx.allcr = ctx->allcr; - - /* - * extra := MPD_EXPDIGITS = MPD_EXP_MAX_T - * wp := prec + 4 + extra - * abs(err) < 5 * 10**-wp - * y := log(base) * exp - * Calculate: - * 1) e**(y * (1 + err)**2) * (1 + err) - * = e**y * e**(y * (2*err + err**2)) * (1 + err) - * ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - * Relative error of the underlined term: - * 2) abs(e**(y * (2*err + err**2)) - 1) - * Case abs(y) >= 10**extra: - * 3) adjexp(y)+1 > log10(abs(y)) >= extra - * This triggers the Overflow/Underflow shortcut in _mpd_qexp(), - * so no further analysis is necessary. - * Case abs(y) < 10**extra: - * 4) abs(y * (2*err + err**2)) < 1/5 * 10**(-prec - 2) - * Use (see _mpd_qexp): - * 5) abs(x) <= 9/10 * 10**-p ==> abs(e**x - 1) < 10**-p - * With 2), 4) and 5): - * 6) abs(e**(y * (2*err + err**2)) - 1) < 10**(-prec - 2) - * The complete relative error of 1) is: - * 7) abs(result - e**y) < e**y * 1/5 * 10**(-prec - 1) - */ - mpd_qln(result, base, &workctx, &workctx.status); - mpd_qmul(result, result, &texp, &workctx, &workctx.status); - mpd_qexp(result, result, &workctx, status); - - mpd_del(&texp); - *status |= (workctx.status&MPD_Errors); - *status |= (MPD_Inexact|MPD_Rounded); -} - -/* The power function: base**exp */ -void -mpd_qpow(mpd_t *result, const mpd_t *base, const mpd_t *exp, - const mpd_context_t *ctx, uint32_t *status) -{ - uint8_t resultsign = 0; - int intexp = 0; - int cmp; - - if (mpd_isspecial(base) || mpd_isspecial(exp)) { - if (mpd_qcheck_nans(result, base, exp, ctx, status)) { - return; - } - } - if (mpd_isinteger(exp)) { - intexp = 1; - resultsign = mpd_isnegative(base) && mpd_isodd(exp); - } - - if (mpd_iszero(base)) { - if (mpd_iszero(exp)) { - mpd_seterror(result, MPD_Invalid_operation, status); - } - else if (mpd_isnegative(exp)) { - mpd_setspecial(result, resultsign, MPD_INF); - } - else { - _settriple(result, resultsign, 0, 0); - } - return; - } - if (mpd_isnegative(base)) { - if (!intexp || mpd_isinfinite(exp)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - } - if (mpd_isinfinite(exp)) { - /* power of one */ - cmp = _qcheck_pow_one_inf(result, base, resultsign, ctx, status); - if (cmp == 0) { - return; - } - else { - cmp *= mpd_arith_sign(exp); - if (cmp < 0) { - _settriple(result, resultsign, 0, 0); - } - else { - mpd_setspecial(result, resultsign, MPD_INF); - } - } - return; - } - if (mpd_isinfinite(base)) { - if (mpd_iszero(exp)) { - _settriple(result, resultsign, 1, 0); - } - else if (mpd_isnegative(exp)) { - _settriple(result, resultsign, 0, 0); - } - else { - mpd_setspecial(result, resultsign, MPD_INF); - } - return; - } - if (mpd_iszero(exp)) { - _settriple(result, resultsign, 1, 0); - return; - } - if (_qcheck_pow_one(result, base, exp, resultsign, ctx, status) == 0) { - return; - } - if (_qcheck_pow_bounds(result, base, exp, resultsign, ctx, status)) { - return; - } - - if (intexp) { - _mpd_qpow_int(result, base, exp, resultsign, ctx, status); - } - else { - _mpd_qpow_real(result, base, exp, ctx, status); - if (!mpd_isspecial(result) && _mpd_cmp(result, &one) == 0) { - mpd_ssize_t shift = ctx->prec-1; - mpd_qshiftl(result, &one, shift, status); - result->exp = -shift; - } - if (mpd_isinfinite(result)) { - /* for ROUND_DOWN, ROUND_FLOOR, etc. */ - _settriple(result, MPD_POS, 1, MPD_EXP_INF); - } - mpd_qfinalize(result, ctx, status); - } -} - -/* - * Internal function: Integer powmod with mpd_uint_t exponent, base is modified! - * Function can fail with MPD_Malloc_error. - */ -static inline void -_mpd_qpowmod_uint(mpd_t *result, mpd_t *base, mpd_uint_t exp, - const mpd_t *mod, uint32_t *status) -{ - mpd_context_t maxcontext; - - mpd_maxcontext(&maxcontext); - - /* resize to smaller cannot fail */ - mpd_qcopy(result, &one, status); - - while (exp > 0) { - if (exp & 1) { - _mpd_qmul_exact(result, result, base, &maxcontext, status); - mpd_qrem(result, result, mod, &maxcontext, status); - } - _mpd_qmul_exact(base, base, base, &maxcontext, status); - mpd_qrem(base, base, mod, &maxcontext, status); - exp >>= 1; - } -} - -/* The powmod function: (base**exp) % mod */ -void -mpd_qpowmod(mpd_t *result, const mpd_t *base, const mpd_t *exp, - const mpd_t *mod, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(tbase,0,0,0,0); - MPD_NEW_STATIC(texp,0,0,0,0); - MPD_NEW_STATIC(tmod,0,0,0,0); - MPD_NEW_STATIC(tmp,0,0,0,0); - MPD_NEW_CONST(two,0,0,1,1,1,2); - mpd_ssize_t tbase_exp, texp_exp; - mpd_ssize_t i; - mpd_t t; - mpd_uint_t r; - uint8_t sign; - - - if (mpd_isspecial(base) || mpd_isspecial(exp) || mpd_isspecial(mod)) { - if (mpd_qcheck_3nans(result, base, exp, mod, ctx, status)) { - return; - } - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - - if (!_mpd_isint(base) || !_mpd_isint(exp) || !_mpd_isint(mod)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_iszerocoeff(mod)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mod->digits+mod->exp > ctx->prec) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - sign = (mpd_isnegative(base)) && (mpd_isodd(exp)); - if (mpd_iszerocoeff(exp)) { - if (mpd_iszerocoeff(base)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - r = (_mpd_cmp_abs(mod, &one)==0) ? 0 : 1; - _settriple(result, sign, r, 0); - return; - } - if (mpd_isnegative(exp)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (mpd_iszerocoeff(base)) { - _settriple(result, sign, 0, 0); - return; - } - - mpd_maxcontext(&maxcontext); - - mpd_qrescale(&tmod, mod, 0, &maxcontext, &maxcontext.status); - if (maxcontext.status&MPD_Errors) { - mpd_seterror(result, maxcontext.status&MPD_Errors, status); - goto out; - } - maxcontext.status = 0; - mpd_set_positive(&tmod); - - mpd_qround_to_int(&tbase, base, &maxcontext, status); - mpd_set_positive(&tbase); - tbase_exp = tbase.exp; - tbase.exp = 0; - - mpd_qround_to_int(&texp, exp, &maxcontext, status); - texp_exp = texp.exp; - texp.exp = 0; - - /* base = (base.int % modulo * pow(10, base.exp, modulo)) % modulo */ - mpd_qrem(&tbase, &tbase, &tmod, &maxcontext, status); - mpd_qshiftl(result, &one, tbase_exp, status); - mpd_qrem(result, result, &tmod, &maxcontext, status); - _mpd_qmul_exact(&tbase, &tbase, result, &maxcontext, status); - mpd_qrem(&tbase, &tbase, &tmod, &maxcontext, status); - if (mpd_isspecial(&tbase) || - mpd_isspecial(&texp) || - mpd_isspecial(&tmod)) { - goto mpd_errors; - } - - for (i = 0; i < texp_exp; i++) { - _mpd_qpowmod_uint(&tmp, &tbase, 10, &tmod, status); - t = tmp; - tmp = tbase; - tbase = t; - } - if (mpd_isspecial(&tbase)) { - goto mpd_errors; /* GCOV_UNLIKELY */ - } - - /* resize to smaller cannot fail */ - mpd_qcopy(result, &one, status); - while (mpd_isfinite(&texp) && !mpd_iszero(&texp)) { - if (mpd_isodd(&texp)) { - _mpd_qmul_exact(result, result, &tbase, &maxcontext, status); - mpd_qrem(result, result, &tmod, &maxcontext, status); - } - _mpd_qmul_exact(&tbase, &tbase, &tbase, &maxcontext, status); - mpd_qrem(&tbase, &tbase, &tmod, &maxcontext, status); - mpd_qdivint(&texp, &texp, &two, &maxcontext, status); - } - if (mpd_isspecial(&texp) || mpd_isspecial(&tbase) || - mpd_isspecial(&tmod) || mpd_isspecial(result)) { - /* MPD_Malloc_error */ - goto mpd_errors; - } - else { - mpd_set_sign(result, sign); - } - -out: - mpd_del(&tbase); - mpd_del(&texp); - mpd_del(&tmod); - mpd_del(&tmp); - return; - -mpd_errors: - mpd_setspecial(result, MPD_POS, MPD_NAN); - goto out; -} - -void -mpd_qquantize(mpd_t *result, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_ssize_t b_exp = b->exp; - mpd_ssize_t expdiff, shift; - mpd_uint_t rnd; - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(result, a, b, ctx, status)) { - return; - } - if (mpd_isinfinite(a) && mpd_isinfinite(b)) { - mpd_qcopy(result, a, status); - return; - } - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - if (b->exp > ctx->emax || b->exp < mpd_etiny(ctx)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - if (mpd_iszero(a)) { - _settriple(result, mpd_sign(a), 0, b->exp); - mpd_qfinalize(result, ctx, status); - return; - } - - - expdiff = a->exp - b->exp; - if (a->digits + expdiff > ctx->prec) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - if (expdiff >= 0) { - shift = expdiff; - if (!mpd_qshiftl(result, a, shift, status)) { - return; - } - result->exp = b_exp; - } - else { - /* At this point expdiff < 0 and a->digits+expdiff <= prec, - * so the shift before an increment will fit in prec. */ - shift = -expdiff; - rnd = mpd_qshiftr(result, a, shift, status); - if (rnd == MPD_UINT_MAX) { - return; - } - result->exp = b_exp; - if (!_mpd_apply_round_fit(result, rnd, ctx, status)) { - return; - } - workstatus |= MPD_Rounded; - if (rnd) { - workstatus |= MPD_Inexact; - } - } - - if (mpd_adjexp(result) > ctx->emax || - mpd_adjexp(result) < mpd_etiny(ctx)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - *status |= workstatus; - mpd_qfinalize(result, ctx, status); -} - -void -mpd_qreduce(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_ssize_t shift, maxexp, maxshift; - uint8_t sign_a = mpd_sign(a); - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - mpd_qcopy(result, a, status); - return; - } - - if (!mpd_qcopy(result, a, status)) { - return; - } - mpd_qfinalize(result, ctx, status); - if (mpd_isspecial(result)) { - return; - } - if (mpd_iszero(result)) { - _settriple(result, sign_a, 0, 0); - return; - } - - shift = mpd_trail_zeros(result); - maxexp = (ctx->clamp) ? mpd_etop(ctx) : ctx->emax; - /* After the finalizing above result->exp <= maxexp. */ - maxshift = maxexp - result->exp; - shift = (shift > maxshift) ? maxshift : shift; - - mpd_qshiftr_inplace(result, shift); - result->exp += shift; -} - -void -mpd_qrem(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, - uint32_t *status) -{ - MPD_NEW_STATIC(q,0,0,0,0); - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(r, a, b, ctx, status)) { - return; - } - if (mpd_isinfinite(a)) { - mpd_seterror(r, MPD_Invalid_operation, status); - return; - } - if (mpd_isinfinite(b)) { - mpd_qcopy(r, a, status); - mpd_qfinalize(r, ctx, status); - return; - } - /* debug */ - abort(); /* GCOV_NOT_REACHED */ - } - if (mpd_iszerocoeff(b)) { - if (mpd_iszerocoeff(a)) { - mpd_seterror(r, MPD_Division_undefined, status); - } - else { - mpd_seterror(r, MPD_Invalid_operation, status); - } - return; - } - - _mpd_qdivmod(&q, r, a, b, ctx, status); - mpd_del(&q); - mpd_qfinalize(r, ctx, status); -} - -void -mpd_qrem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_context_t workctx; - MPD_NEW_STATIC(btmp,0,0,0,0); - MPD_NEW_STATIC(q,0,0,0,0); - mpd_ssize_t expdiff, qdigits; - int cmp, isodd, allnine; - - assert(r != NULL); /* annotation for scan-build */ - - if (mpd_isspecial(a) || mpd_isspecial(b)) { - if (mpd_qcheck_nans(r, a, b, ctx, status)) { - return; - } - if (mpd_isinfinite(a)) { - mpd_seterror(r, MPD_Invalid_operation, status); - return; - } - if (mpd_isinfinite(b)) { - mpd_qcopy(r, a, status); - mpd_qfinalize(r, ctx, status); - return; - } - /* debug */ - abort(); /* GCOV_NOT_REACHED */ - } - if (mpd_iszerocoeff(b)) { - if (mpd_iszerocoeff(a)) { - mpd_seterror(r, MPD_Division_undefined, status); - } - else { - mpd_seterror(r, MPD_Invalid_operation, status); - } - return; - } - - if (r == b) { - if (!mpd_qcopy(&btmp, b, status)) { - mpd_seterror(r, MPD_Malloc_error, status); - return; - } - b = &btmp; - } - - _mpd_qdivmod(&q, r, a, b, ctx, status); - if (mpd_isnan(&q) || mpd_isnan(r)) { - goto finish; - } - if (mpd_iszerocoeff(r)) { - goto finish; - } - - expdiff = mpd_adjexp(b) - mpd_adjexp(r); - if (-1 <= expdiff && expdiff <= 1) { - - allnine = mpd_coeff_isallnine(&q); - qdigits = q.digits; - isodd = mpd_isodd(&q); - - mpd_maxcontext(&workctx); - if (mpd_sign(a) == mpd_sign(b)) { - /* sign(r) == sign(b) */ - _mpd_qsub(&q, r, b, &workctx, &workctx.status); - } - else { - /* sign(r) != sign(b) */ - _mpd_qadd(&q, r, b, &workctx, &workctx.status); - } - - if (workctx.status&MPD_Errors) { - mpd_seterror(r, workctx.status&MPD_Errors, status); - goto finish; - } - - cmp = _mpd_cmp_abs(&q, r); - if (cmp < 0 || (cmp == 0 && isodd)) { - /* abs(r) > abs(b)/2 or abs(r) == abs(b)/2 and isodd(quotient) */ - if (allnine && qdigits == ctx->prec) { - /* abs(quotient) + 1 == 10**prec */ - mpd_seterror(r, MPD_Division_impossible, status); - goto finish; - } - mpd_qcopy(r, &q, status); - } - } - - -finish: - mpd_del(&btmp); - mpd_del(&q); - mpd_qfinalize(r, ctx, status); -} - -static void -_mpd_qrescale(mpd_t *result, const mpd_t *a, mpd_ssize_t exp, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t expdiff, shift; - mpd_uint_t rnd; - - if (mpd_isspecial(a)) { - mpd_qcopy(result, a, status); - return; - } - - if (mpd_iszero(a)) { - _settriple(result, mpd_sign(a), 0, exp); - return; - } - - expdiff = a->exp - exp; - if (expdiff >= 0) { - shift = expdiff; - if (a->digits + shift > MPD_MAX_PREC+1) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - if (!mpd_qshiftl(result, a, shift, status)) { - return; - } - result->exp = exp; - } - else { - shift = -expdiff; - rnd = mpd_qshiftr(result, a, shift, status); - if (rnd == MPD_UINT_MAX) { - return; - } - result->exp = exp; - _mpd_apply_round_excess(result, rnd, ctx, status); - *status |= MPD_Rounded; - if (rnd) { - *status |= MPD_Inexact; - } - } - - if (mpd_issubnormal(result, ctx)) { - *status |= MPD_Subnormal; - } -} - -/* - * Rescale a number so that it has exponent 'exp'. Does not regard context - * precision, emax, emin, but uses the rounding mode. Special numbers are - * quietly copied. Restrictions: - * - * MPD_MIN_ETINY <= exp <= MPD_MAX_EMAX+1 - * result->digits <= MPD_MAX_PREC+1 - */ -void -mpd_qrescale(mpd_t *result, const mpd_t *a, mpd_ssize_t exp, - const mpd_context_t *ctx, uint32_t *status) -{ - if (exp > MPD_MAX_EMAX+1 || exp < MPD_MIN_ETINY) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - _mpd_qrescale(result, a, exp, ctx, status); -} - -/* - * Same as mpd_qrescale, but with relaxed restrictions. The result of this - * function should only be used for formatting a number and never as input - * for other operations. - * - * MPD_MIN_ETINY-MPD_MAX_PREC <= exp <= MPD_MAX_EMAX+1 - * result->digits <= MPD_MAX_PREC+1 - */ -void -mpd_qrescale_fmt(mpd_t *result, const mpd_t *a, mpd_ssize_t exp, - const mpd_context_t *ctx, uint32_t *status) -{ - if (exp > MPD_MAX_EMAX+1 || exp < MPD_MIN_ETINY-MPD_MAX_PREC) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - _mpd_qrescale(result, a, exp, ctx, status); -} - -/* Round to an integer according to 'action' and ctx->round. */ -enum {TO_INT_EXACT, TO_INT_SILENT, TO_INT_TRUNC}; -static void -_mpd_qround_to_integral(int action, mpd_t *result, const mpd_t *a, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_uint_t rnd; - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - mpd_qcopy(result, a, status); - return; - } - if (a->exp >= 0) { - mpd_qcopy(result, a, status); - return; - } - if (mpd_iszerocoeff(a)) { - _settriple(result, mpd_sign(a), 0, 0); - return; - } - - rnd = mpd_qshiftr(result, a, -a->exp, status); - if (rnd == MPD_UINT_MAX) { - return; - } - result->exp = 0; - - if (action == TO_INT_EXACT || action == TO_INT_SILENT) { - _mpd_apply_round_excess(result, rnd, ctx, status); - if (action == TO_INT_EXACT) { - *status |= MPD_Rounded; - if (rnd) { - *status |= MPD_Inexact; - } - } - } -} - -void -mpd_qround_to_intx(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - (void)_mpd_qround_to_integral(TO_INT_EXACT, result, a, ctx, status); -} - -void -mpd_qround_to_int(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - (void)_mpd_qround_to_integral(TO_INT_SILENT, result, a, ctx, status); -} - -void -mpd_qtrunc(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - if (mpd_isspecial(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - (void)_mpd_qround_to_integral(TO_INT_TRUNC, result, a, ctx, status); -} - -void -mpd_qfloor(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx = *ctx; - - if (mpd_isspecial(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - workctx.round = MPD_ROUND_FLOOR; - (void)_mpd_qround_to_integral(TO_INT_SILENT, result, a, - &workctx, status); -} - -void -mpd_qceil(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx = *ctx; - - if (mpd_isspecial(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - workctx.round = MPD_ROUND_CEILING; - (void)_mpd_qround_to_integral(TO_INT_SILENT, result, a, - &workctx, status); -} - -int -mpd_same_quantum(const mpd_t *a, const mpd_t *b) -{ - if (mpd_isspecial(a) || mpd_isspecial(b)) { - return ((mpd_isnan(a) && mpd_isnan(b)) || - (mpd_isinfinite(a) && mpd_isinfinite(b))); - } - - return a->exp == b->exp; -} - -/* Schedule the increase in precision for the Newton iteration. */ -static inline int -recpr_schedule_prec(mpd_ssize_t klist[MPD_MAX_PREC_LOG2], - mpd_ssize_t maxprec, mpd_ssize_t initprec) -{ - mpd_ssize_t k; - int i; - - assert(maxprec > 0 && initprec > 0); - if (maxprec <= initprec) return -1; - - i = 0; k = maxprec; - do { - k = (k+1) / 2; - klist[i++] = k; - } while (k > initprec); - - return i-1; -} - -/* - * Initial approximation for the reciprocal: - * k_0 := MPD_RDIGITS-2 - * z_0 := 10**(-k_0) * floor(10**(2*k_0 + 2) / floor(v * 10**(k_0 + 2))) - * Absolute error: - * |1/v - z_0| < 10**(-k_0) - * ACL2 proof: maxerror-inverse-approx - */ -static void -_mpd_qreciprocal_approx(mpd_t *z, const mpd_t *v, uint32_t *status) -{ - mpd_uint_t p10data[2] = {0, mpd_pow10[MPD_RDIGITS-2]}; - mpd_uint_t dummy, word; - int n; - - assert(v->exp == -v->digits); - - _mpd_get_msdigits(&dummy, &word, v, MPD_RDIGITS); - n = mpd_word_digits(word); - word *= mpd_pow10[MPD_RDIGITS-n]; - - mpd_qresize(z, 2, status); - (void)_mpd_shortdiv(z->data, p10data, 2, word); - - mpd_clear_flags(z); - z->exp = -(MPD_RDIGITS-2); - z->len = (z->data[1] == 0) ? 1 : 2; - mpd_setdigits(z); -} - -/* - * Reciprocal, calculated with Newton's Method. Assumption: result != a. - * NOTE: The comments in the function show that certain operations are - * exact. The proof for the maximum error is too long to fit in here. - * ACL2 proof: maxerror-inverse-complete - */ -static void -_mpd_qreciprocal(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t varcontext, maxcontext; - mpd_t *z = result; /* current approximation */ - mpd_t *v; /* a, normalized to a number between 0.1 and 1 */ - MPD_NEW_SHARED(vtmp, a); /* v shares data with a */ - MPD_NEW_STATIC(s,0,0,0,0); /* temporary variable */ - MPD_NEW_STATIC(t,0,0,0,0); /* temporary variable */ - MPD_NEW_CONST(two,0,0,1,1,1,2); /* const 2 */ - mpd_ssize_t klist[MPD_MAX_PREC_LOG2]; - mpd_ssize_t adj, maxprec, initprec; - uint8_t sign = mpd_sign(a); - int i; - - assert(result != a); - - v = &vtmp; - mpd_clear_flags(v); - adj = v->digits + v->exp; - v->exp = -v->digits; - - /* Initial approximation */ - _mpd_qreciprocal_approx(z, v, status); - - mpd_maxcontext(&varcontext); - mpd_maxcontext(&maxcontext); - varcontext.round = maxcontext.round = MPD_ROUND_TRUNC; - varcontext.emax = maxcontext.emax = MPD_MAX_EMAX + 100; - varcontext.emin = maxcontext.emin = MPD_MIN_EMIN - 100; - maxcontext.prec = MPD_MAX_PREC + 100; - - maxprec = ctx->prec; - maxprec += 2; - initprec = MPD_RDIGITS-3; - - i = recpr_schedule_prec(klist, maxprec, initprec); - for (; i >= 0; i--) { - /* Loop invariant: z->digits <= klist[i]+7 */ - /* Let s := z**2, exact result */ - _mpd_qmul_exact(&s, z, z, &maxcontext, status); - varcontext.prec = 2*klist[i] + 5; - if (v->digits > varcontext.prec) { - /* Let t := v, truncated to n >= 2*k+5 fraction digits */ - mpd_qshiftr(&t, v, v->digits-varcontext.prec, status); - t.exp = -varcontext.prec; - /* Let t := trunc(v)*s, truncated to n >= 2*k+1 fraction digits */ - mpd_qmul(&t, &t, &s, &varcontext, status); - } - else { /* v->digits <= 2*k+5 */ - /* Let t := v*s, truncated to n >= 2*k+1 fraction digits */ - mpd_qmul(&t, v, &s, &varcontext, status); - } - /* Let s := 2*z, exact result */ - _mpd_qmul_exact(&s, z, &two, &maxcontext, status); - /* s.digits < t.digits <= 2*k+5, |adjexp(s)-adjexp(t)| <= 1, - * so the subtraction generates at most 2*k+6 <= klist[i+1]+7 - * digits. The loop invariant is preserved. */ - _mpd_qsub_exact(z, &s, &t, &maxcontext, status); - } - - if (!mpd_isspecial(z)) { - z->exp -= adj; - mpd_set_flags(z, sign); - } - - mpd_del(&s); - mpd_del(&t); - mpd_qfinalize(z, ctx, status); -} - -/* - * Internal function for large numbers: - * - * q, r = divmod(coeff(a), coeff(b)) - * - * Strategy: Multiply the dividend by the reciprocal of the divisor. The - * inexact result is fixed by a small loop, using at most one iteration. - * - * ACL2 proofs: - * ------------ - * 1) q is a natural number. (ndivmod-quotient-natp) - * 2) r is a natural number. (ndivmod-remainder-natp) - * 3) a = q * b + r (ndivmod-q*b+r==a) - * 4) r < b (ndivmod-remainder-<-b) - */ -static void -_mpd_base_ndivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, - uint32_t *status) -{ - mpd_context_t workctx; - mpd_t *qq = q, *rr = r; - mpd_t aa, bb; - int k; - - _mpd_copy_shared(&aa, a); - _mpd_copy_shared(&bb, b); - - mpd_set_positive(&aa); - mpd_set_positive(&bb); - aa.exp = 0; - bb.exp = 0; - - if (q == a || q == b) { - if ((qq = mpd_qnew()) == NULL) { - *status |= MPD_Malloc_error; - goto nanresult; - } - } - if (r == a || r == b) { - if ((rr = mpd_qnew()) == NULL) { - *status |= MPD_Malloc_error; - goto nanresult; - } - } - - mpd_maxcontext(&workctx); - - /* Let prec := adigits - bdigits + 4 */ - workctx.prec = a->digits - b->digits + 1 + 3; - if (a->digits > MPD_MAX_PREC || workctx.prec > MPD_MAX_PREC) { - *status |= MPD_Division_impossible; - goto nanresult; - } - - /* Let x := _mpd_qreciprocal(b, prec) - * Then x is bounded by: - * 1) 1/b - 10**(-prec - bdigits) < x < 1/b + 10**(-prec - bdigits) - * 2) 1/b - 10**(-adigits - 4) < x < 1/b + 10**(-adigits - 4) - */ - _mpd_qreciprocal(rr, &bb, &workctx, &workctx.status); - - /* Get an estimate for the quotient. Let q := a * x - * Then q is bounded by: - * 3) a/b - 10**-4 < q < a/b + 10**-4 - */ - _mpd_qmul(qq, &aa, rr, &workctx, &workctx.status); - /* Truncate q to an integer: - * 4) a/b - 2 < trunc(q) < a/b + 1 - */ - mpd_qtrunc(qq, qq, &workctx, &workctx.status); - - workctx.prec = aa.digits + 3; - workctx.emax = MPD_MAX_EMAX + 3; - workctx.emin = MPD_MIN_EMIN - 3; - /* Multiply the estimate for q by b: - * 5) a - 2 * b < trunc(q) * b < a + b - */ - _mpd_qmul(rr, &bb, qq, &workctx, &workctx.status); - /* Get the estimate for r such that a = q * b + r. */ - _mpd_qsub_exact(rr, &aa, rr, &workctx, &workctx.status); - - /* Fix the result. At this point -b < r < 2*b, so the correction loop - takes at most one iteration. */ - for (k = 0;; k++) { - if (mpd_isspecial(qq) || mpd_isspecial(rr)) { - *status |= (workctx.status&MPD_Errors); - goto nanresult; - } - if (k > 2) { /* Allow two iterations despite the proof. */ - mpd_err_warn("libmpdec: internal error in " /* GCOV_NOT_REACHED */ - "_mpd_base_ndivmod: please report"); /* GCOV_NOT_REACHED */ - *status |= MPD_Invalid_operation; /* GCOV_NOT_REACHED */ - goto nanresult; /* GCOV_NOT_REACHED */ - } - /* r < 0 */ - else if (_mpd_cmp(&zero, rr) == 1) { - _mpd_qadd_exact(rr, rr, &bb, &workctx, &workctx.status); - _mpd_qadd_exact(qq, qq, &minus_one, &workctx, &workctx.status); - } - /* 0 <= r < b */ - else if (_mpd_cmp(rr, &bb) == -1) { - break; - } - /* r >= b */ - else { - _mpd_qsub_exact(rr, rr, &bb, &workctx, &workctx.status); - _mpd_qadd_exact(qq, qq, &one, &workctx, &workctx.status); - } - } - - if (qq != q) { - if (!mpd_qcopy(q, qq, status)) { - goto nanresult; /* GCOV_UNLIKELY */ - } - mpd_del(qq); - } - if (rr != r) { - if (!mpd_qcopy(r, rr, status)) { - goto nanresult; /* GCOV_UNLIKELY */ - } - mpd_del(rr); - } - - *status |= (workctx.status&MPD_Errors); - return; - - -nanresult: - if (qq && qq != q) mpd_del(qq); - if (rr && rr != r) mpd_del(rr); - mpd_setspecial(q, MPD_POS, MPD_NAN); - mpd_setspecial(r, MPD_POS, MPD_NAN); -} - -/* LIBMPDEC_ONLY */ -/* - * Schedule the optimal precision increase for the Newton iteration. - * v := input operand - * z_0 := initial approximation - * initprec := natural number such that abs(sqrt(v) - z_0) < 10**-initprec - * maxprec := target precision - * - * For convenience the output klist contains the elements in reverse order: - * klist := [k_n-1, ..., k_0], where - * 1) k_0 <= initprec and - * 2) abs(sqrt(v) - result) < 10**(-2*k_n-1 + 2) <= 10**-maxprec. - */ -static inline int -invroot_schedule_prec(mpd_ssize_t klist[MPD_MAX_PREC_LOG2], - mpd_ssize_t maxprec, mpd_ssize_t initprec) -{ - mpd_ssize_t k; - int i; - - assert(maxprec >= 3 && initprec >= 3); - if (maxprec <= initprec) return -1; - - i = 0; k = maxprec; - do { - k = (k+3) / 2; - klist[i++] = k; - } while (k > initprec); - - return i-1; -} - -/* - * Initial approximation for the inverse square root function. - * Input: - * v := rational number, with 1 <= v < 100 - * vhat := floor(v * 10**6) - * Output: - * z := approximation to 1/sqrt(v), such that abs(z - 1/sqrt(v)) < 10**-3. - */ -static inline void -_invroot_init_approx(mpd_t *z, mpd_uint_t vhat) -{ - mpd_uint_t lo = 1000; - mpd_uint_t hi = 10000; - mpd_uint_t a, sq; - - assert(lo*lo <= vhat && vhat < (hi+1)*(hi+1)); - - for(;;) { - a = (lo + hi) / 2; - sq = a * a; - if (vhat >= sq) { - if (vhat < sq + 2*a + 1) { - break; - } - lo = a + 1; - } - else { - hi = a - 1; - } - } - - /* - * After the binary search we have: - * 1) a**2 <= floor(v * 10**6) < (a + 1)**2 - * This implies: - * 2) a**2 <= v * 10**6 < (a + 1)**2 - * 3) a <= sqrt(v) * 10**3 < a + 1 - * Since 10**3 <= a: - * 4) 0 <= 10**prec/a - 1/sqrt(v) < 10**-prec - * We have: - * 5) 10**3/a - 10**-3 < floor(10**9/a) * 10**-6 <= 10**3/a - * Merging 4) and 5): - * 6) abs(floor(10**9/a) * 10**-6 - 1/sqrt(v)) < 10**-3 - */ - mpd_minalloc(z); - mpd_clear_flags(z); - z->data[0] = 1000000000UL / a; - z->len = 1; - z->exp = -6; - mpd_setdigits(z); -} - -/* - * Set 'result' to 1/sqrt(a). - * Relative error: abs(result - 1/sqrt(a)) < 10**-prec * 1/sqrt(a) - */ -static void -_mpd_qinvroot(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - uint32_t workstatus = 0; - mpd_context_t varcontext, maxcontext; - mpd_t *z = result; /* current approximation */ - mpd_t *v; /* a, normalized to a number between 1 and 100 */ - MPD_NEW_SHARED(vtmp, a); /* by default v will share data with a */ - MPD_NEW_STATIC(s,0,0,0,0); /* temporary variable */ - MPD_NEW_STATIC(t,0,0,0,0); /* temporary variable */ - MPD_NEW_CONST(one_half,0,-1,1,1,1,5); - MPD_NEW_CONST(three,0,0,1,1,1,3); - mpd_ssize_t klist[MPD_MAX_PREC_LOG2]; - mpd_ssize_t ideal_exp, shift; - mpd_ssize_t adj, tz; - mpd_ssize_t maxprec, fracdigits; - mpd_uint_t vhat, dummy; - int i, n; - - - ideal_exp = -(a->exp - (a->exp & 1)) / 2; - - v = &vtmp; - if (result == a) { - if ((v = mpd_qncopy(a)) == NULL) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - } - - /* normalize a to 1 <= v < 100 */ - if ((v->digits+v->exp) & 1) { - fracdigits = v->digits - 1; - v->exp = -fracdigits; - n = (v->digits > 7) ? 7 : (int)v->digits; - /* Let vhat := floor(v * 10**(2*initprec)) */ - _mpd_get_msdigits(&dummy, &vhat, v, n); - if (n < 7) { - vhat *= mpd_pow10[7-n]; - } - } - else { - fracdigits = v->digits - 2; - v->exp = -fracdigits; - n = (v->digits > 8) ? 8 : (int)v->digits; - /* Let vhat := floor(v * 10**(2*initprec)) */ - _mpd_get_msdigits(&dummy, &vhat, v, n); - if (n < 8) { - vhat *= mpd_pow10[8-n]; - } - } - adj = (a->exp-v->exp) / 2; - - /* initial approximation */ - _invroot_init_approx(z, vhat); - - mpd_maxcontext(&maxcontext); - mpd_maxcontext(&varcontext); - varcontext.round = MPD_ROUND_TRUNC; - maxprec = ctx->prec + 1; - - /* initprec == 3 */ - i = invroot_schedule_prec(klist, maxprec, 3); - for (; i >= 0; i--) { - varcontext.prec = 2*klist[i]+2; - mpd_qmul(&s, z, z, &maxcontext, &workstatus); - if (v->digits > varcontext.prec) { - shift = v->digits - varcontext.prec; - mpd_qshiftr(&t, v, shift, &workstatus); - t.exp += shift; - mpd_qmul(&t, &t, &s, &varcontext, &workstatus); - } - else { - mpd_qmul(&t, v, &s, &varcontext, &workstatus); - } - mpd_qsub(&t, &three, &t, &maxcontext, &workstatus); - mpd_qmul(z, z, &t, &varcontext, &workstatus); - mpd_qmul(z, z, &one_half, &maxcontext, &workstatus); - } - - z->exp -= adj; - - tz = mpd_trail_zeros(result); - shift = ideal_exp - result->exp; - shift = (tz > shift) ? shift : tz; - if (shift > 0) { - mpd_qshiftr_inplace(result, shift); - result->exp += shift; - } - - - mpd_del(&s); - mpd_del(&t); - if (v != &vtmp) mpd_del(v); - *status |= (workstatus&MPD_Errors); - *status |= (MPD_Rounded|MPD_Inexact); -} - -void -mpd_qinvroot(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t workctx; - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - if (mpd_isnegative(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - /* positive infinity */ - _settriple(result, MPD_POS, 0, mpd_etiny(ctx)); - *status |= MPD_Clamped; - return; - } - if (mpd_iszero(a)) { - mpd_setspecial(result, mpd_sign(a), MPD_INF); - *status |= MPD_Division_by_zero; - return; - } - if (mpd_isnegative(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - workctx = *ctx; - workctx.prec += 2; - workctx.round = MPD_ROUND_HALF_EVEN; - _mpd_qinvroot(result, a, &workctx, status); - mpd_qfinalize(result, ctx, status); -} -/* END LIBMPDEC_ONLY */ - -/* Algorithm from decimal.py */ -static void -_mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - mpd_context_t maxcontext; - MPD_NEW_STATIC(c,0,0,0,0); - MPD_NEW_STATIC(q,0,0,0,0); - MPD_NEW_STATIC(r,0,0,0,0); - MPD_NEW_CONST(two,0,0,1,1,1,2); - mpd_ssize_t prec, ideal_exp; - mpd_ssize_t l, shift; - int exact = 0; - - - ideal_exp = (a->exp - (a->exp & 1)) / 2; - - if (mpd_isspecial(a)) { - if (mpd_qcheck_nan(result, a, ctx, status)) { - return; - } - if (mpd_isnegative(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - mpd_setspecial(result, MPD_POS, MPD_INF); - return; - } - if (mpd_iszero(a)) { - _settriple(result, mpd_sign(a), 0, ideal_exp); - mpd_qfinalize(result, ctx, status); - return; - } - if (mpd_isnegative(a)) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - mpd_maxcontext(&maxcontext); - prec = ctx->prec + 1; - - if (!mpd_qcopy(&c, a, status)) { - goto malloc_error; - } - c.exp = 0; - - if (a->exp & 1) { - if (!mpd_qshiftl(&c, &c, 1, status)) { - goto malloc_error; - } - l = (a->digits >> 1) + 1; - } - else { - l = (a->digits + 1) >> 1; - } - - shift = prec - l; - if (shift >= 0) { - if (!mpd_qshiftl(&c, &c, 2*shift, status)) { - goto malloc_error; - } - exact = 1; - } - else { - exact = !mpd_qshiftr_inplace(&c, -2*shift); - } - - ideal_exp -= shift; - - /* find result = floor(sqrt(c)) using Newton's method */ - if (!mpd_qshiftl(result, &one, prec, status)) { - goto malloc_error; - } - - while (1) { - _mpd_qdivmod(&q, &r, &c, result, &maxcontext, &maxcontext.status); - if (mpd_isspecial(result) || mpd_isspecial(&q)) { - mpd_seterror(result, maxcontext.status&MPD_Errors, status); - goto out; - } - if (_mpd_cmp(result, &q) <= 0) { - break; - } - _mpd_qadd_exact(result, result, &q, &maxcontext, &maxcontext.status); - if (mpd_isspecial(result)) { - mpd_seterror(result, maxcontext.status&MPD_Errors, status); - goto out; - } - _mpd_qdivmod(result, &r, result, &two, &maxcontext, &maxcontext.status); - } - - if (exact) { - _mpd_qmul_exact(&r, result, result, &maxcontext, &maxcontext.status); - if (mpd_isspecial(&r)) { - mpd_seterror(result, maxcontext.status&MPD_Errors, status); - goto out; - } - exact = (_mpd_cmp(&r, &c) == 0); - } - - if (exact) { - if (shift >= 0) { - mpd_qshiftr_inplace(result, shift); - } - else { - if (!mpd_qshiftl(result, result, -shift, status)) { - goto malloc_error; - } - } - ideal_exp += shift; - } - else { - int lsd = (int)mpd_lsd(result->data[0]); - if (lsd == 0 || lsd == 5) { - result->data[0] += 1; - } - } - - result->exp = ideal_exp; - - -out: - mpd_del(&c); - mpd_del(&q); - mpd_del(&r); - maxcontext = *ctx; - maxcontext.round = MPD_ROUND_HALF_EVEN; - mpd_qfinalize(result, &maxcontext, status); - return; - -malloc_error: - mpd_seterror(result, MPD_Malloc_error, status); - goto out; -} - -void -mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, - uint32_t *status) -{ - MPD_NEW_STATIC(aa,0,0,0,0); - uint32_t xstatus = 0; - - if (result == a) { - if (!mpd_qcopy(&aa, a, status)) { - mpd_seterror(result, MPD_Malloc_error, status); - goto out; - } - a = &aa; - } - - _mpd_qsqrt(result, a, ctx, &xstatus); - - if (xstatus & (MPD_Malloc_error|MPD_Division_impossible)) { - /* The above conditions can occur at very high context precisions - * if intermediate values get too large. Retry the operation with - * a lower context precision in case the result is exact. - * - * If the result is exact, an upper bound for the number of digits - * is the number of digits in the input. - * - * NOTE: sqrt(40e9) = 2.0e+5 /\ digits(40e9) = digits(2.0e+5) = 2 - */ - uint32_t ystatus = 0; - mpd_context_t workctx = *ctx; - - workctx.prec = a->digits; - if (workctx.prec >= ctx->prec) { - *status |= (xstatus|MPD_Errors); - goto out; /* No point in repeating this, keep the original error. */ - } - - _mpd_qsqrt(result, a, &workctx, &ystatus); - if (ystatus != 0) { - ystatus = *status | ((xstatus|ystatus)&MPD_Errors); - mpd_seterror(result, ystatus, status); - } - } - else { - *status |= xstatus; - } - -out: - mpd_del(&aa); -} - - -/******************************************************************************/ -/* Base conversions */ -/******************************************************************************/ - -/* Space needed to represent an integer mpd_t in base 'base'. */ -size_t -mpd_sizeinbase(const mpd_t *a, uint32_t base) -{ - double x; - size_t digits; - double upper_bound; - - assert(mpd_isinteger(a)); - assert(base >= 2); - - if (mpd_iszero(a)) { - return 1; - } - - digits = a->digits+a->exp; - -#ifdef CONFIG_64 - /* ceil(2711437152599294 / log10(2)) + 4 == 2**53 */ - if (digits > 2711437152599294ULL) { - return SIZE_MAX; - } - - upper_bound = (double)((1ULL<<53)-1); -#else - upper_bound = (double)(SIZE_MAX-1); -#endif - - x = (double)digits / log10(base); - return (x > upper_bound) ? SIZE_MAX : (size_t)x + 1; -} - -/* Space needed to import a base 'base' integer of length 'srclen'. */ -static mpd_ssize_t -_mpd_importsize(size_t srclen, uint32_t base) -{ - double x; - double upper_bound; - - assert(srclen > 0); - assert(base >= 2); - -#if SIZE_MAX == UINT64_MAX - if (srclen > (1ULL<<53)) { - return MPD_SSIZE_MAX; - } - - assert((1ULL<<53) <= MPD_MAXIMPORT); - upper_bound = (double)((1ULL<<53)-1); -#else - upper_bound = MPD_MAXIMPORT-1; -#endif - - x = (double)srclen * (log10(base)/MPD_RDIGITS); - return (x > upper_bound) ? MPD_SSIZE_MAX : (mpd_ssize_t)x + 1; -} - -static uint8_t -mpd_resize_u16(uint16_t **w, size_t nmemb) -{ - uint8_t err = 0; - *w = mpd_realloc(*w, nmemb, sizeof **w, &err); - return !err; -} - -static uint8_t -mpd_resize_u32(uint32_t **w, size_t nmemb) -{ - uint8_t err = 0; - *w = mpd_realloc(*w, nmemb, sizeof **w, &err); - return !err; -} - -static size_t -_baseconv_to_u16(uint16_t **w, size_t wlen, mpd_uint_t wbase, - mpd_uint_t *u, mpd_ssize_t ulen) -{ - size_t n = 0; - - assert(wlen > 0 && ulen > 0); - assert(wbase <= (1U<<16)); - - do { - if (n >= wlen) { - if (!mpd_resize_u16(w, n+1)) { - return SIZE_MAX; - } - wlen = n+1; - } - (*w)[n++] = (uint16_t)_mpd_shortdiv(u, u, ulen, wbase); - /* ulen is at least 1. u[ulen-1] can only be zero if ulen == 1. */ - ulen = _mpd_real_size(u, ulen); - - } while (u[ulen-1] != 0); - - return n; -} - -static size_t -_coeff_from_u16(mpd_t *w, mpd_ssize_t wlen, - const mpd_uint_t *u, size_t ulen, uint32_t ubase, - uint32_t *status) -{ - mpd_ssize_t n = 0; - mpd_uint_t carry; - - assert(wlen > 0 && ulen > 0); - assert(ubase <= (1U<<16)); - - w->data[n++] = u[--ulen]; - while (--ulen != SIZE_MAX) { - carry = _mpd_shortmul_c(w->data, w->data, n, ubase); - if (carry) { - if (n >= wlen) { - if (!mpd_qresize(w, n+1, status)) { - return SIZE_MAX; - } - wlen = n+1; - } - w->data[n++] = carry; - } - carry = _mpd_shortadd(w->data, n, u[ulen]); - if (carry) { - if (n >= wlen) { - if (!mpd_qresize(w, n+1, status)) { - return SIZE_MAX; - } - wlen = n+1; - } - w->data[n++] = carry; - } - } - - return n; -} - -/* target base wbase < source base ubase */ -static size_t -_baseconv_to_smaller(uint32_t **w, size_t wlen, uint32_t wbase, - mpd_uint_t *u, mpd_ssize_t ulen, mpd_uint_t ubase) -{ - size_t n = 0; - - assert(wlen > 0 && ulen > 0); - assert(wbase < ubase); - - do { - if (n >= wlen) { - if (!mpd_resize_u32(w, n+1)) { - return SIZE_MAX; - } - wlen = n+1; - } - (*w)[n++] = (uint32_t)_mpd_shortdiv_b(u, u, ulen, wbase, ubase); - /* ulen is at least 1. u[ulen-1] can only be zero if ulen == 1. */ - ulen = _mpd_real_size(u, ulen); - - } while (u[ulen-1] != 0); - - return n; -} - -#ifdef CONFIG_32 -/* target base 'wbase' == source base 'ubase' */ -static size_t -_copy_equal_base(uint32_t **w, size_t wlen, - const uint32_t *u, size_t ulen) -{ - if (wlen < ulen) { - if (!mpd_resize_u32(w, ulen)) { - return SIZE_MAX; - } - } - - memcpy(*w, u, ulen * (sizeof **w)); - return ulen; -} - -/* target base 'wbase' > source base 'ubase' */ -static size_t -_baseconv_to_larger(uint32_t **w, size_t wlen, mpd_uint_t wbase, - const mpd_uint_t *u, size_t ulen, mpd_uint_t ubase) -{ - size_t n = 0; - mpd_uint_t carry; - - assert(wlen > 0 && ulen > 0); - assert(ubase < wbase); - - (*w)[n++] = u[--ulen]; - while (--ulen != SIZE_MAX) { - carry = _mpd_shortmul_b(*w, *w, n, ubase, wbase); - if (carry) { - if (n >= wlen) { - if (!mpd_resize_u32(w, n+1)) { - return SIZE_MAX; - } - wlen = n+1; - } - (*w)[n++] = carry; - } - carry = _mpd_shortadd_b(*w, n, u[ulen], wbase); - if (carry) { - if (n >= wlen) { - if (!mpd_resize_u32(w, n+1)) { - return SIZE_MAX; - } - wlen = n+1; - } - (*w)[n++] = carry; - } - } - - return n; -} - -/* target base wbase < source base ubase */ -static size_t -_coeff_from_larger_base(mpd_t *w, size_t wlen, mpd_uint_t wbase, - mpd_uint_t *u, mpd_ssize_t ulen, mpd_uint_t ubase, - uint32_t *status) -{ - size_t n = 0; - - assert(wlen > 0 && ulen > 0); - assert(wbase < ubase); - - do { - if (n >= wlen) { - if (!mpd_qresize(w, n+1, status)) { - return SIZE_MAX; - } - wlen = n+1; - } - w->data[n++] = (uint32_t)_mpd_shortdiv_b(u, u, ulen, wbase, ubase); - /* ulen is at least 1. u[ulen-1] can only be zero if ulen == 1. */ - ulen = _mpd_real_size(u, ulen); - - } while (u[ulen-1] != 0); - - return n; -} -#endif - -/* target base 'wbase' > source base 'ubase' */ -static size_t -_coeff_from_smaller_base(mpd_t *w, mpd_ssize_t wlen, mpd_uint_t wbase, - const uint32_t *u, size_t ulen, mpd_uint_t ubase, - uint32_t *status) -{ - mpd_ssize_t n = 0; - mpd_uint_t carry; - - assert(wlen > 0 && ulen > 0); - assert(wbase > ubase); - - w->data[n++] = u[--ulen]; - while (--ulen != SIZE_MAX) { - carry = _mpd_shortmul_b(w->data, w->data, n, ubase, wbase); - if (carry) { - if (n >= wlen) { - if (!mpd_qresize(w, n+1, status)) { - return SIZE_MAX; - } - wlen = n+1; - } - w->data[n++] = carry; - } - carry = _mpd_shortadd_b(w->data, n, u[ulen], wbase); - if (carry) { - if (n >= wlen) { - if (!mpd_qresize(w, n+1, status)) { - return SIZE_MAX; - } - wlen = n+1; - } - w->data[n++] = carry; - } - } - - return n; -} - -/* - * Convert an integer mpd_t to a multiprecision integer with base <= 2**16. - * The least significant word of the result is (*rdata)[0]. - * - * If rdata is NULL, space is allocated by the function and rlen is irrelevant. - * In case of an error any allocated storage is freed and rdata is set back to - * NULL. - * - * If rdata is non-NULL, it MUST be allocated by one of libmpdec's allocation - * functions and rlen MUST be correct. If necessary, the function will resize - * rdata. In case of an error the caller must free rdata. - * - * Return value: In case of success, the exact length of rdata, SIZE_MAX - * otherwise. - */ -size_t -mpd_qexport_u16(uint16_t **rdata, size_t rlen, uint32_t rbase, - const mpd_t *src, uint32_t *status) -{ - MPD_NEW_STATIC(tsrc,0,0,0,0); - int alloc = 0; /* rdata == NULL */ - size_t n; - - assert(rbase <= (1U<<16)); - - if (mpd_isspecial(src) || !_mpd_isint(src)) { - *status |= MPD_Invalid_operation; - return SIZE_MAX; - } - - if (*rdata == NULL) { - rlen = mpd_sizeinbase(src, rbase); - if (rlen == SIZE_MAX) { - *status |= MPD_Invalid_operation; - return SIZE_MAX; - } - *rdata = mpd_alloc(rlen, sizeof **rdata); - if (*rdata == NULL) { - goto malloc_error; - } - alloc = 1; - } - - if (mpd_iszero(src)) { - **rdata = 0; - return 1; - } - - if (src->exp >= 0) { - if (!mpd_qshiftl(&tsrc, src, src->exp, status)) { - goto malloc_error; - } - } - else { - if (mpd_qshiftr(&tsrc, src, -src->exp, status) == MPD_UINT_MAX) { - goto malloc_error; - } - } - - n = _baseconv_to_u16(rdata, rlen, rbase, tsrc.data, tsrc.len); - if (n == SIZE_MAX) { - goto malloc_error; - } - - -out: - mpd_del(&tsrc); - return n; - -malloc_error: - if (alloc) { - mpd_free(*rdata); - *rdata = NULL; - } - n = SIZE_MAX; - *status |= MPD_Malloc_error; - goto out; -} - -/* - * Convert an integer mpd_t to a multiprecision integer with base<=UINT32_MAX. - * The least significant word of the result is (*rdata)[0]. - * - * If rdata is NULL, space is allocated by the function and rlen is irrelevant. - * In case of an error any allocated storage is freed and rdata is set back to - * NULL. - * - * If rdata is non-NULL, it MUST be allocated by one of libmpdec's allocation - * functions and rlen MUST be correct. If necessary, the function will resize - * rdata. In case of an error the caller must free rdata. - * - * Return value: In case of success, the exact length of rdata, SIZE_MAX - * otherwise. - */ -size_t -mpd_qexport_u32(uint32_t **rdata, size_t rlen, uint32_t rbase, - const mpd_t *src, uint32_t *status) -{ - MPD_NEW_STATIC(tsrc,0,0,0,0); - int alloc = 0; /* rdata == NULL */ - size_t n; - - if (mpd_isspecial(src) || !_mpd_isint(src)) { - *status |= MPD_Invalid_operation; - return SIZE_MAX; - } - - if (*rdata == NULL) { - rlen = mpd_sizeinbase(src, rbase); - if (rlen == SIZE_MAX) { - *status |= MPD_Invalid_operation; - return SIZE_MAX; - } - *rdata = mpd_alloc(rlen, sizeof **rdata); - if (*rdata == NULL) { - goto malloc_error; - } - alloc = 1; - } - - if (mpd_iszero(src)) { - **rdata = 0; - return 1; - } - - if (src->exp >= 0) { - if (!mpd_qshiftl(&tsrc, src, src->exp, status)) { - goto malloc_error; - } - } - else { - if (mpd_qshiftr(&tsrc, src, -src->exp, status) == MPD_UINT_MAX) { - goto malloc_error; - } - } - -#ifdef CONFIG_64 - n = _baseconv_to_smaller(rdata, rlen, rbase, - tsrc.data, tsrc.len, MPD_RADIX); -#else - if (rbase == MPD_RADIX) { - n = _copy_equal_base(rdata, rlen, tsrc.data, tsrc.len); - } - else if (rbase < MPD_RADIX) { - n = _baseconv_to_smaller(rdata, rlen, rbase, - tsrc.data, tsrc.len, MPD_RADIX); - } - else { - n = _baseconv_to_larger(rdata, rlen, rbase, - tsrc.data, tsrc.len, MPD_RADIX); - } -#endif - - if (n == SIZE_MAX) { - goto malloc_error; - } - - -out: - mpd_del(&tsrc); - return n; - -malloc_error: - if (alloc) { - mpd_free(*rdata); - *rdata = NULL; - } - n = SIZE_MAX; - *status |= MPD_Malloc_error; - goto out; -} - - -/* - * Converts a multiprecision integer with base <= UINT16_MAX+1 to an mpd_t. - * The least significant word of the source is srcdata[0]. - */ -void -mpd_qimport_u16(mpd_t *result, - const uint16_t *srcdata, size_t srclen, - uint8_t srcsign, uint32_t srcbase, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_uint_t *usrc; /* uint16_t src copied to an mpd_uint_t array */ - mpd_ssize_t rlen; /* length of the result */ - size_t n; - - assert(srclen > 0); - assert(srcbase <= (1U<<16)); - - rlen = _mpd_importsize(srclen, srcbase); - if (rlen == MPD_SSIZE_MAX) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - usrc = mpd_alloc((mpd_size_t)srclen, sizeof *usrc); - if (usrc == NULL) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - for (n = 0; n < srclen; n++) { - usrc[n] = srcdata[n]; - } - - if (!mpd_qresize(result, rlen, status)) { - goto finish; - } - - n = _coeff_from_u16(result, rlen, usrc, srclen, srcbase, status); - if (n == SIZE_MAX) { - goto finish; - } - - mpd_set_flags(result, srcsign); - result->exp = 0; - result->len = n; - mpd_setdigits(result); - - mpd_qresize(result, result->len, status); - mpd_qfinalize(result, ctx, status); - - -finish: - mpd_free(usrc); -} - -/* - * Converts a multiprecision integer with base <= UINT32_MAX to an mpd_t. - * The least significant word of the source is srcdata[0]. - */ -void -mpd_qimport_u32(mpd_t *result, - const uint32_t *srcdata, size_t srclen, - uint8_t srcsign, uint32_t srcbase, - const mpd_context_t *ctx, uint32_t *status) -{ - mpd_ssize_t rlen; /* length of the result */ - size_t n; - - assert(srclen > 0); - - rlen = _mpd_importsize(srclen, srcbase); - if (rlen == MPD_SSIZE_MAX) { - mpd_seterror(result, MPD_Invalid_operation, status); - return; - } - - if (!mpd_qresize(result, rlen, status)) { - return; - } - -#ifdef CONFIG_64 - n = _coeff_from_smaller_base(result, rlen, MPD_RADIX, - srcdata, srclen, srcbase, - status); -#else - if (srcbase == MPD_RADIX) { - if (!mpd_qresize(result, srclen, status)) { - return; - } - memcpy(result->data, srcdata, srclen * (sizeof *srcdata)); - n = srclen; - } - else if (srcbase < MPD_RADIX) { - n = _coeff_from_smaller_base(result, rlen, MPD_RADIX, - srcdata, srclen, srcbase, - status); - } - else { - mpd_uint_t *usrc = mpd_alloc((mpd_size_t)srclen, sizeof *usrc); - if (usrc == NULL) { - mpd_seterror(result, MPD_Malloc_error, status); - return; - } - for (n = 0; n < srclen; n++) { - usrc[n] = srcdata[n]; - } - - n = _coeff_from_larger_base(result, rlen, MPD_RADIX, - usrc, (mpd_ssize_t)srclen, srcbase, - status); - mpd_free(usrc); - } -#endif - - if (n == SIZE_MAX) { - return; - } - - mpd_set_flags(result, srcsign); - result->exp = 0; - result->len = n; - mpd_setdigits(result); - - mpd_qresize(result, result->len, status); - mpd_qfinalize(result, ctx, status); -} - - -/******************************************************************************/ -/* From triple */ -/******************************************************************************/ - -#if defined(CONFIG_64) && defined(__SIZEOF_INT128__) -static mpd_ssize_t -_set_coeff(uint64_t data[3], uint64_t hi, uint64_t lo) -{ - __uint128_t d = ((__uint128_t)hi << 64) + lo; - __uint128_t q, r; - - q = d / MPD_RADIX; - r = d % MPD_RADIX; - data[0] = (uint64_t)r; - d = q; - - q = d / MPD_RADIX; - r = d % MPD_RADIX; - data[1] = (uint64_t)r; - d = q; - - q = d / MPD_RADIX; - r = d % MPD_RADIX; - data[2] = (uint64_t)r; - - if (q != 0) { - abort(); /* GCOV_NOT_REACHED */ - } - - return data[2] != 0 ? 3 : (data[1] != 0 ? 2 : 1); -} -#else -static size_t -_uint_from_u16(mpd_uint_t *w, mpd_ssize_t wlen, const uint16_t *u, size_t ulen) -{ - const mpd_uint_t ubase = 1U<<16; - mpd_ssize_t n = 0; - mpd_uint_t carry; - - assert(wlen > 0 && ulen > 0); - - w[n++] = u[--ulen]; - while (--ulen != SIZE_MAX) { - carry = _mpd_shortmul_c(w, w, n, ubase); - if (carry) { - if (n >= wlen) { - abort(); /* GCOV_NOT_REACHED */ - } - w[n++] = carry; - } - carry = _mpd_shortadd(w, n, u[ulen]); - if (carry) { - if (n >= wlen) { - abort(); /* GCOV_NOT_REACHED */ - } - w[n++] = carry; - } - } - - return n; -} - -static mpd_ssize_t -_set_coeff(mpd_uint_t *data, mpd_ssize_t len, uint64_t hi, uint64_t lo) -{ - uint16_t u16[8] = {0}; - - u16[7] = (uint16_t)((hi & 0xFFFF000000000000ULL) >> 48); - u16[6] = (uint16_t)((hi & 0x0000FFFF00000000ULL) >> 32); - u16[5] = (uint16_t)((hi & 0x00000000FFFF0000ULL) >> 16); - u16[4] = (uint16_t) (hi & 0x000000000000FFFFULL); - - u16[3] = (uint16_t)((lo & 0xFFFF000000000000ULL) >> 48); - u16[2] = (uint16_t)((lo & 0x0000FFFF00000000ULL) >> 32); - u16[1] = (uint16_t)((lo & 0x00000000FFFF0000ULL) >> 16); - u16[0] = (uint16_t) (lo & 0x000000000000FFFFULL); - - return (mpd_ssize_t)_uint_from_u16(data, len, u16, 8); -} -#endif - -static int -_set_uint128_coeff_exp(mpd_t *result, uint64_t hi, uint64_t lo, mpd_ssize_t exp) -{ - mpd_uint_t data[5] = {0}; - uint32_t status = 0; - mpd_ssize_t len; - -#if defined(CONFIG_64) && defined(__SIZEOF_INT128__) - len = _set_coeff(data, hi, lo); -#else - len = _set_coeff(data, 5, hi, lo); -#endif - - if (!mpd_qresize(result, len, &status)) { - return -1; - } - - for (mpd_ssize_t i = 0; i < len; i++) { - result->data[i] = data[i]; - } - - result->exp = exp; - result->len = len; - mpd_setdigits(result); - - return 0; -} - -int -mpd_from_uint128_triple(mpd_t *result, const mpd_uint128_triple_t *triple, uint32_t *status) -{ - static const mpd_context_t maxcontext = { - .prec=MPD_MAX_PREC, - .emax=MPD_MAX_EMAX, - .emin=MPD_MIN_EMIN, - .round=MPD_ROUND_HALF_EVEN, - .traps=MPD_Traps, - .status=0, - .newtrap=0, - .clamp=0, - .allcr=1, - }; - const enum mpd_triple_class tag = triple->tag; - const uint8_t sign = triple->sign; - const uint64_t hi = triple->hi; - const uint64_t lo = triple->lo; - mpd_ssize_t exp; - -#ifdef CONFIG_32 - if (triple->exp < MPD_SSIZE_MIN || triple->exp > MPD_SSIZE_MAX) { - goto conversion_error; - } -#endif - exp = (mpd_ssize_t)triple->exp; - - switch (tag) { - case MPD_TRIPLE_QNAN: case MPD_TRIPLE_SNAN: { - if (sign > 1 || exp != 0) { - goto conversion_error; - } - - const uint8_t flags = tag == MPD_TRIPLE_QNAN ? MPD_NAN : MPD_SNAN; - mpd_setspecial(result, sign, flags); - - if (hi == 0 && lo == 0) { /* no payload */ - return 0; - } - - if (_set_uint128_coeff_exp(result, hi, lo, exp) < 0) { - goto malloc_error; - } - - return 0; - } - - case MPD_TRIPLE_INF: { - if (sign > 1 || hi != 0 || lo != 0 || exp != 0) { - goto conversion_error; - } - - mpd_setspecial(result, sign, MPD_INF); - - return 0; - } - - case MPD_TRIPLE_NORMAL: { - if (sign > 1) { - goto conversion_error; - } - - const uint8_t flags = sign ? MPD_NEG : MPD_POS; - mpd_set_flags(result, flags); - - if (exp > MPD_EXP_INF) { - exp = MPD_EXP_INF; - } - if (exp == MPD_SSIZE_MIN) { - exp = MPD_SSIZE_MIN+1; - } - - if (_set_uint128_coeff_exp(result, hi, lo, exp) < 0) { - goto malloc_error; - } - - uint32_t workstatus = 0; - mpd_qfinalize(result, &maxcontext, &workstatus); - if (workstatus & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { - goto conversion_error; - } - - return 0; - } - - default: - goto conversion_error; - } - -conversion_error: - mpd_seterror(result, MPD_Conversion_syntax, status); - return -1; - -malloc_error: - mpd_seterror(result, MPD_Malloc_error, status); - return -1; -} - - -/******************************************************************************/ -/* As triple */ -/******************************************************************************/ - -#if defined(CONFIG_64) && defined(__SIZEOF_INT128__) -static void -_get_coeff(uint64_t *hi, uint64_t *lo, const mpd_t *a) -{ - __uint128_t u128 = 0; - - switch (a->len) { - case 3: - u128 = a->data[2]; /* fall through */ - case 2: - u128 = u128 * MPD_RADIX + a->data[1]; /* fall through */ - case 1: - u128 = u128 * MPD_RADIX + a->data[0]; - break; - default: - abort(); /* GCOV_NOT_REACHED */ - } - - *hi = u128 >> 64; - *lo = (uint64_t)u128; -} -#else -static size_t -_uint_to_u16(uint16_t w[8], mpd_uint_t *u, mpd_ssize_t ulen) -{ - const mpd_uint_t wbase = 1U<<16; - size_t n = 0; - - assert(ulen > 0); - - do { - if (n >= 8) { - abort(); /* GCOV_NOT_REACHED */ - } - w[n++] = (uint16_t)_mpd_shortdiv(u, u, ulen, wbase); - /* ulen is at least 1. u[ulen-1] can only be zero if ulen == 1. */ - ulen = _mpd_real_size(u, ulen); - - } while (u[ulen-1] != 0); - - return n; -} - -static void -_get_coeff(uint64_t *hi, uint64_t *lo, const mpd_t *a) -{ - uint16_t u16[8] = {0}; - mpd_uint_t data[5] = {0}; - - switch (a->len) { - case 5: - data[4] = a->data[4]; /* fall through */ - case 4: - data[3] = a->data[3]; /* fall through */ - case 3: - data[2] = a->data[2]; /* fall through */ - case 2: - data[1] = a->data[1]; /* fall through */ - case 1: - data[0] = a->data[0]; - break; - default: - abort(); /* GCOV_NOT_REACHED */ - } - - _uint_to_u16(u16, data, a->len); - - *hi = (uint64_t)u16[7] << 48; - *hi |= (uint64_t)u16[6] << 32; - *hi |= (uint64_t)u16[5] << 16; - *hi |= (uint64_t)u16[4]; - - *lo = (uint64_t)u16[3] << 48; - *lo |= (uint64_t)u16[2] << 32; - *lo |= (uint64_t)u16[1] << 16; - *lo |= (uint64_t)u16[0]; -} -#endif - -static enum mpd_triple_class -_coeff_as_uint128(uint64_t *hi, uint64_t *lo, const mpd_t *a) -{ -#ifdef CONFIG_64 - static mpd_uint_t uint128_max_data[3] = { 3374607431768211455ULL, 4028236692093846346ULL, 3ULL }; - static const mpd_t uint128_max = { MPD_STATIC|MPD_CONST_DATA, 0, 39, 3, 3, uint128_max_data }; -#else - static mpd_uint_t uint128_max_data[5] = { 768211455U, 374607431U, 938463463U, 282366920U, 340U }; - static const mpd_t uint128_max = { MPD_STATIC|MPD_CONST_DATA, 0, 39, 5, 5, uint128_max_data }; -#endif - enum mpd_triple_class ret = MPD_TRIPLE_NORMAL; - uint32_t status = 0; - mpd_t coeff; - - *hi = *lo = 0ULL; - - if (mpd_isspecial(a)) { - if (mpd_isinfinite(a)) { - return MPD_TRIPLE_INF; - } - - ret = mpd_isqnan(a) ? MPD_TRIPLE_QNAN : MPD_TRIPLE_SNAN; - if (a->len == 0) { /* no payload */ - return ret; - } - } - else if (mpd_iszero(a)) { - return ret; - } - - _mpd_copy_shared(&coeff, a); - mpd_set_flags(&coeff, 0); - coeff.exp = 0; - - if (mpd_qcmp(&coeff, &uint128_max, &status) > 0) { - return MPD_TRIPLE_ERROR; - } - - _get_coeff(hi, lo, &coeff); - return ret; -} - -mpd_uint128_triple_t -mpd_as_uint128_triple(const mpd_t *a) -{ - mpd_uint128_triple_t triple = { MPD_TRIPLE_ERROR, 0, 0, 0, 0 }; - - triple.tag = _coeff_as_uint128(&triple.hi, &triple.lo, a); - if (triple.tag == MPD_TRIPLE_ERROR) { - return triple; - } - - triple.sign = !!mpd_isnegative(a); - if (triple.tag == MPD_TRIPLE_NORMAL) { - triple.exp = a->exp; - } - - return triple; -} diff --git a/Modules/_decimal/libmpdec/mpdecimal.h b/Modules/_decimal/libmpdec/mpdecimal.h deleted file mode 100644 index 24c280b00ebcd00..000000000000000 --- a/Modules/_decimal/libmpdec/mpdecimal.h +++ /dev/null @@ -1,847 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_MPDECIMAL_H_ -#define LIBMPDEC_MPDECIMAL_H_ - - -#ifndef _MSC_VER - #include "pyconfig.h" -#endif - -#ifdef __cplusplus - #include <cinttypes> - #include <climits> - #include <cstdint> - #include <cstdio> - #include <cstdlib> - #define MPD_UINT8_C(x) (static_cast<uint8_t>(x)) -extern "C" { -#else - #include <inttypes.h> - #include <limits.h> - #include <stdint.h> - #include <stdio.h> - #include <stdlib.h> - #define MPD_UINT8_C(x) ((uint8_t)x) -#endif - - -#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)) && \ - defined(__GNUC__) && __GNUC__ >= 4 && !defined(__INTEL_COMPILER) - #define MPD_PRAGMA(x) _Pragma(x) - #define MPD_HIDE_SYMBOLS_START "GCC visibility push(hidden)" - #define MPD_HIDE_SYMBOLS_END "GCC visibility pop" -#else - #define MPD_PRAGMA(x) - #define MPD_HIDE_SYMBOLS_START - #define MPD_HIDE_SYMBOLS_END -#endif - -#if defined(_MSC_VER) - #define EXTINLINE extern inline -#else - #define EXTINLINE -#endif - - -/* This header file is internal for the purpose of building _decimal.so. - * All symbols should have local scope in the DSO. */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -/******************************************************************************/ -/* Version */ -/******************************************************************************/ - -#define MPD_MAJOR_VERSION 2 -#define MPD_MINOR_VERSION 5 -#define MPD_MICRO_VERSION 1 - -#define MPD_VERSION "2.5.1" - -#define MPD_VERSION_HEX ((MPD_MAJOR_VERSION << 24) | \ - (MPD_MINOR_VERSION << 16) | \ - (MPD_MICRO_VERSION << 8)) - -const char *mpd_version(void); - - -/******************************************************************************/ -/* Configuration */ -/******************************************************************************/ - -#if defined(UNIVERSAL) - #if defined(CONFIG_64) || defined(CONFIG_32) - #error "cannot use CONFIG_64 or CONFIG_32 with UNIVERSAL." - #endif - #if defined(__ppc__) - #define CONFIG_32 - #define ANSI - #elif defined(__ppc64__) - #define CONFIG_64 - #define ANSI - #elif defined(__i386__) - #define CONFIG_32 - #define ANSI - #elif defined(__x86_64__) - #define CONFIG_64 - #define ASM - #elif defined(__arm64__) - #define CONFIG_64 - #define ANSI - #else - #error "unknown architecture for universal build." - #endif -#endif - - -/* BEGIN CONFIG_64 */ -#if defined(CONFIG_64) -/* types for modular and base arithmetic */ -#define MPD_UINT_MAX UINT64_MAX -#define MPD_BITS_PER_UINT 64 -typedef uint64_t mpd_uint_t; /* unsigned mod type */ - -#define MPD_SIZE_MAX SIZE_MAX -typedef size_t mpd_size_t; /* unsigned size type */ - -/* type for exp, digits, len, prec */ -#define MPD_SSIZE_MAX INT64_MAX -#define MPD_SSIZE_MIN INT64_MIN -typedef int64_t mpd_ssize_t; -#define _mpd_strtossize strtoll - -/* decimal arithmetic */ -#define MPD_RADIX 10000000000000000000ULL /* 10**19 */ -#define MPD_RDIGITS 19 -#define MPD_MAX_POW10 19 -#define MPD_EXPDIGITS 19 /* MPD_EXPDIGITS <= MPD_RDIGITS+1 */ - -#define MPD_MAXTRANSFORM_2N 4294967296ULL /* 2**32 */ -#define MPD_MAX_PREC 999999999999999999LL -#define MPD_MAX_PREC_LOG2 64 -#define MPD_ELIMIT 1000000000000000000LL -#define MPD_MAX_EMAX 999999999999999999LL /* ELIMIT-1 */ -#define MPD_MIN_EMIN (-999999999999999999LL) /* -EMAX */ -#define MPD_MIN_ETINY (MPD_MIN_EMIN-(MPD_MAX_PREC-1)) -#define MPD_EXP_INF 2000000000000000001LL -#define MPD_EXP_CLAMP (-4000000000000000001LL) -#define MPD_MAXIMPORT 105263157894736842L /* ceil((2*MPD_MAX_PREC)/MPD_RDIGITS) */ -#define MPD_IEEE_CONTEXT_MAX_BITS 512 /* 16*(log2(MPD_MAX_EMAX / 3)-3) */ - -/* conversion specifiers */ -#define PRI_mpd_uint_t PRIu64 -#define PRI_mpd_ssize_t PRIi64 -/* END CONFIG_64 */ - - -/* BEGIN CONFIG_32 */ -#elif defined(CONFIG_32) -/* types for modular and base arithmetic */ -#define MPD_UINT_MAX UINT32_MAX -#define MPD_BITS_PER_UINT 32 -typedef uint32_t mpd_uint_t; /* unsigned mod type */ - -#ifndef LEGACY_COMPILER -#define MPD_UUINT_MAX UINT64_MAX -typedef uint64_t mpd_uuint_t; /* double width unsigned mod type */ -#endif - -#define MPD_SIZE_MAX SIZE_MAX -typedef size_t mpd_size_t; /* unsigned size type */ - -/* type for dec->len, dec->exp, ctx->prec */ -#define MPD_SSIZE_MAX INT32_MAX -#define MPD_SSIZE_MIN INT32_MIN -typedef int32_t mpd_ssize_t; -#define _mpd_strtossize strtol - -/* decimal arithmetic */ -#define MPD_RADIX 1000000000UL /* 10**9 */ -#define MPD_RDIGITS 9 -#define MPD_MAX_POW10 9 -#define MPD_EXPDIGITS 10 /* MPD_EXPDIGITS <= MPD_RDIGITS+1 */ - -#define MPD_MAXTRANSFORM_2N 33554432UL /* 2**25 */ -#define MPD_MAX_PREC 425000000L -#define MPD_MAX_PREC_LOG2 32 -#define MPD_ELIMIT 425000001L -#define MPD_MAX_EMAX 425000000L /* ELIMIT-1 */ -#define MPD_MIN_EMIN (-425000000L) /* -EMAX */ -#define MPD_MIN_ETINY (MPD_MIN_EMIN-(MPD_MAX_PREC-1)) -#define MPD_EXP_INF 1000000001L /* allows for emax=999999999 in the tests */ -#define MPD_EXP_CLAMP (-2000000001L) /* allows for emin=-999999999 in the tests */ -#define MPD_MAXIMPORT 94444445L /* ceil((2*MPD_MAX_PREC)/MPD_RDIGITS) */ -#define MPD_IEEE_CONTEXT_MAX_BITS 256 /* 16*(log2(MPD_MAX_EMAX / 3)-3) */ - -/* conversion specifiers */ -#define PRI_mpd_uint_t PRIu32 -#define PRI_mpd_ssize_t PRIi32 -/* END CONFIG_32 */ - -#else - #error "define CONFIG_64 or CONFIG_32" -#endif -/* END CONFIG */ - - -#if MPD_SIZE_MAX != MPD_UINT_MAX - #error "unsupported platform: need mpd_size_t == mpd_uint_t" -#endif - - -/******************************************************************************/ -/* Context */ -/******************************************************************************/ - -enum { - MPD_ROUND_UP, /* round away from 0 */ - MPD_ROUND_DOWN, /* round toward 0 (truncate) */ - MPD_ROUND_CEILING, /* round toward +infinity */ - MPD_ROUND_FLOOR, /* round toward -infinity */ - MPD_ROUND_HALF_UP, /* 0.5 is rounded up */ - MPD_ROUND_HALF_DOWN, /* 0.5 is rounded down */ - MPD_ROUND_HALF_EVEN, /* 0.5 is rounded to even */ - MPD_ROUND_05UP, /* round zero or five away from 0 */ - MPD_ROUND_TRUNC, /* truncate, but set infinity */ - MPD_ROUND_GUARD -}; - -enum { MPD_CLAMP_DEFAULT, MPD_CLAMP_IEEE_754, MPD_CLAMP_GUARD }; - -extern const char * const mpd_round_string[MPD_ROUND_GUARD]; -extern const char * const mpd_clamp_string[MPD_CLAMP_GUARD]; - - -typedef struct mpd_context_t { - mpd_ssize_t prec; /* precision */ - mpd_ssize_t emax; /* max positive exp */ - mpd_ssize_t emin; /* min negative exp */ - uint32_t traps; /* status events that should be trapped */ - uint32_t status; /* status flags */ - uint32_t newtrap; /* set by mpd_addstatus_raise() */ - int round; /* rounding mode */ - int clamp; /* clamp mode */ - int allcr; /* all functions correctly rounded */ -} mpd_context_t; - - -/* Status flags */ -#define MPD_Clamped 0x00000001U -#define MPD_Conversion_syntax 0x00000002U -#define MPD_Division_by_zero 0x00000004U -#define MPD_Division_impossible 0x00000008U -#define MPD_Division_undefined 0x00000010U -#define MPD_Fpu_error 0x00000020U -#define MPD_Inexact 0x00000040U -#define MPD_Invalid_context 0x00000080U -#define MPD_Invalid_operation 0x00000100U -#define MPD_Malloc_error 0x00000200U -#define MPD_Not_implemented 0x00000400U -#define MPD_Overflow 0x00000800U -#define MPD_Rounded 0x00001000U -#define MPD_Subnormal 0x00002000U -#define MPD_Underflow 0x00004000U -#define MPD_Max_status (0x00008000U-1U) - -/* Conditions that result in an IEEE 754 exception */ -#define MPD_IEEE_Invalid_operation (MPD_Conversion_syntax | \ - MPD_Division_impossible | \ - MPD_Division_undefined | \ - MPD_Fpu_error | \ - MPD_Invalid_context | \ - MPD_Invalid_operation | \ - MPD_Malloc_error) \ - -/* Errors that require the result of an operation to be set to NaN */ -#define MPD_Errors (MPD_IEEE_Invalid_operation | \ - MPD_Division_by_zero) - -/* Default traps */ -#define MPD_Traps (MPD_IEEE_Invalid_operation | \ - MPD_Division_by_zero | \ - MPD_Overflow | \ - MPD_Underflow) - -/* Official name */ -#define MPD_Insufficient_storage MPD_Malloc_error - -/* IEEE 754 interchange format contexts */ -#define MPD_DECIMAL32 32 -#define MPD_DECIMAL64 64 -#define MPD_DECIMAL128 128 - - -#define MPD_MINALLOC_MIN 2 -#define MPD_MINALLOC_MAX 64 -extern mpd_ssize_t MPD_MINALLOC; -extern void (* mpd_traphandler)(mpd_context_t *); -void mpd_dflt_traphandler(mpd_context_t *); - -void mpd_setminalloc(mpd_ssize_t n); -void mpd_init(mpd_context_t *ctx, mpd_ssize_t prec); - -void mpd_maxcontext(mpd_context_t *ctx); -void mpd_defaultcontext(mpd_context_t *ctx); -void mpd_basiccontext(mpd_context_t *ctx); -int mpd_ieee_context(mpd_context_t *ctx, int bits); - -mpd_ssize_t mpd_getprec(const mpd_context_t *ctx); -mpd_ssize_t mpd_getemax(const mpd_context_t *ctx); -mpd_ssize_t mpd_getemin(const mpd_context_t *ctx); -int mpd_getround(const mpd_context_t *ctx); -uint32_t mpd_gettraps(const mpd_context_t *ctx); -uint32_t mpd_getstatus(const mpd_context_t *ctx); -int mpd_getclamp(const mpd_context_t *ctx); -int mpd_getcr(const mpd_context_t *ctx); - -int mpd_qsetprec(mpd_context_t *ctx, mpd_ssize_t prec); -int mpd_qsetemax(mpd_context_t *ctx, mpd_ssize_t emax); -int mpd_qsetemin(mpd_context_t *ctx, mpd_ssize_t emin); -int mpd_qsetround(mpd_context_t *ctx, int newround); -int mpd_qsettraps(mpd_context_t *ctx, uint32_t flags); -int mpd_qsetstatus(mpd_context_t *ctx, uint32_t flags); -int mpd_qsetclamp(mpd_context_t *ctx, int c); -int mpd_qsetcr(mpd_context_t *ctx, int c); -void mpd_addstatus_raise(mpd_context_t *ctx, uint32_t flags); - - -/******************************************************************************/ -/* Decimal Arithmetic */ -/******************************************************************************/ - -/* mpd_t flags */ -#define MPD_POS MPD_UINT8_C(0) -#define MPD_NEG MPD_UINT8_C(1) -#define MPD_INF MPD_UINT8_C(2) -#define MPD_NAN MPD_UINT8_C(4) -#define MPD_SNAN MPD_UINT8_C(8) -#define MPD_SPECIAL (MPD_INF|MPD_NAN|MPD_SNAN) -#define MPD_STATIC MPD_UINT8_C(16) -#define MPD_STATIC_DATA MPD_UINT8_C(32) -#define MPD_SHARED_DATA MPD_UINT8_C(64) -#define MPD_CONST_DATA MPD_UINT8_C(128) -#define MPD_DATAFLAGS (MPD_STATIC_DATA|MPD_SHARED_DATA|MPD_CONST_DATA) - -/* mpd_t */ -typedef struct mpd_t { - uint8_t flags; - mpd_ssize_t exp; - mpd_ssize_t digits; - mpd_ssize_t len; - mpd_ssize_t alloc; - mpd_uint_t *data; -} mpd_t; - - -/******************************************************************************/ -/* Triple */ -/******************************************************************************/ - -/* status cases for getting a triple */ -enum mpd_triple_class { - MPD_TRIPLE_NORMAL, - MPD_TRIPLE_INF, - MPD_TRIPLE_QNAN, - MPD_TRIPLE_SNAN, - MPD_TRIPLE_ERROR, -}; - -typedef struct { - enum mpd_triple_class tag; - uint8_t sign; - uint64_t hi; - uint64_t lo; - int64_t exp; -} mpd_uint128_triple_t; - -int mpd_from_uint128_triple(mpd_t *result, const mpd_uint128_triple_t *triple, uint32_t *status); -mpd_uint128_triple_t mpd_as_uint128_triple(const mpd_t *a); - - -/******************************************************************************/ -/* Quiet, thread-safe functions */ -/******************************************************************************/ - -/* format specification */ -typedef struct mpd_spec_t { - mpd_ssize_t min_width; /* minimum field width */ - mpd_ssize_t prec; /* fraction digits or significant digits */ - char type; /* conversion specifier */ - char align; /* alignment */ - char sign; /* sign printing/alignment */ - char fill[5]; /* fill character */ - const char *dot; /* decimal point */ - const char *sep; /* thousands separator */ - const char *grouping; /* grouping of digits */ -} mpd_spec_t; - -/* output to a string */ -char *mpd_to_sci(const mpd_t *dec, int fmt); -char *mpd_to_eng(const mpd_t *dec, int fmt); -mpd_ssize_t mpd_to_sci_size(char **res, const mpd_t *dec, int fmt); -mpd_ssize_t mpd_to_eng_size(char **res, const mpd_t *dec, int fmt); -int mpd_validate_lconv(mpd_spec_t *spec); -int mpd_parse_fmt_str(mpd_spec_t *spec, const char *fmt, int caps); -char *mpd_qformat_spec(const mpd_t *dec, const mpd_spec_t *spec, const mpd_context_t *ctx, uint32_t *status); -char *mpd_qformat(const mpd_t *dec, const char *fmt, const mpd_context_t *ctx, uint32_t *status); - -#define MPD_NUM_FLAGS 15 -#define MPD_MAX_FLAG_STRING 208 -#define MPD_MAX_FLAG_LIST (MPD_MAX_FLAG_STRING+18) -#define MPD_MAX_SIGNAL_LIST 121 -int mpd_snprint_flags(char *dest, int nmemb, uint32_t flags); -int mpd_lsnprint_flags(char *dest, int nmemb, uint32_t flags, const char *flag_string[]); -int mpd_lsnprint_signals(char *dest, int nmemb, uint32_t flags, const char *signal_string[]); - -/* output to a file */ -void mpd_fprint(FILE *file, const mpd_t *dec); -void mpd_print(const mpd_t *dec); - -/* assignment from a string */ -void mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, uint32_t *status); -void mpd_qset_string_exact(mpd_t *dec, const char *s, uint32_t *status); - -/* set to NaN with error flags */ -void mpd_seterror(mpd_t *result, uint32_t flags, uint32_t *status); -/* set a special with sign and type */ -void mpd_setspecial(mpd_t *result, uint8_t sign, uint8_t type); -/* set coefficient to zero or all nines */ -void mpd_zerocoeff(mpd_t *result); -void mpd_qmaxcoeff(mpd_t *result, const mpd_context_t *ctx, uint32_t *status); - -/* quietly assign a C integer type to an mpd_t */ -void mpd_qset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qset_i32(mpd_t *result, int32_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qset_uint(mpd_t *result, mpd_uint_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qset_u32(mpd_t *result, uint32_t a, const mpd_context_t *ctx, uint32_t *status); -#ifndef LEGACY_COMPILER -void mpd_qset_i64(mpd_t *result, int64_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qset_u64(mpd_t *result, uint64_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qset_i64_exact(mpd_t *result, int64_t a, uint32_t *status); -void mpd_qset_u64_exact(mpd_t *result, uint64_t a, uint32_t *status); -#endif - -/* quietly assign a C integer type to an mpd_t with a static coefficient */ -void mpd_qsset_ssize(mpd_t *result, mpd_ssize_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsset_i32(mpd_t *result, int32_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsset_uint(mpd_t *result, mpd_uint_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsset_u32(mpd_t *result, uint32_t a, const mpd_context_t *ctx, uint32_t *status); - -/* quietly get a C integer type from an mpd_t */ -mpd_ssize_t mpd_qget_ssize(const mpd_t *dec, uint32_t *status); -mpd_uint_t mpd_qget_uint(const mpd_t *dec, uint32_t *status); -mpd_uint_t mpd_qabs_uint(const mpd_t *dec, uint32_t *status); - -int32_t mpd_qget_i32(const mpd_t *dec, uint32_t *status); -uint32_t mpd_qget_u32(const mpd_t *dec, uint32_t *status); -#ifndef LEGACY_COMPILER -int64_t mpd_qget_i64(const mpd_t *dec, uint32_t *status); -uint64_t mpd_qget_u64(const mpd_t *dec, uint32_t *status); -#endif - -/* quiet functions */ -int mpd_qcheck_nan(mpd_t *nanresult, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -int mpd_qcheck_nans(mpd_t *nanresult, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qfinalize(mpd_t *result, const mpd_context_t *ctx, uint32_t *status); - -const char *mpd_class(const mpd_t *a, const mpd_context_t *ctx); - -int mpd_qcopy(mpd_t *result, const mpd_t *a, uint32_t *status); -int mpd_qcopy_cxx(mpd_t *result, const mpd_t *a); -mpd_t *mpd_qncopy(const mpd_t *a); -int mpd_qcopy_abs(mpd_t *result, const mpd_t *a, uint32_t *status); -int mpd_qcopy_negate(mpd_t *result, const mpd_t *a, uint32_t *status); -int mpd_qcopy_sign(mpd_t *result, const mpd_t *a, const mpd_t *b, uint32_t *status); - -void mpd_qand(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qinvert(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qlogb(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qscaleb(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qxor(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -int mpd_same_quantum(const mpd_t *a, const mpd_t *b); - -void mpd_qrotate(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -int mpd_qshiftl(mpd_t *result, const mpd_t *a, mpd_ssize_t n, uint32_t *status); -mpd_uint_t mpd_qshiftr(mpd_t *result, const mpd_t *a, mpd_ssize_t n, uint32_t *status); -mpd_uint_t mpd_qshiftr_inplace(mpd_t *result, mpd_ssize_t n); -void mpd_qshift(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qshiftn(mpd_t *result, const mpd_t *a, mpd_ssize_t n, const mpd_context_t *ctx, uint32_t *status); - -int mpd_qcmp(const mpd_t *a, const mpd_t *b, uint32_t *status); -int mpd_qcompare(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -int mpd_qcompare_signal(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -int mpd_cmp_total(const mpd_t *a, const mpd_t *b); -int mpd_cmp_total_mag(const mpd_t *a, const mpd_t *b); -int mpd_compare_total(mpd_t *result, const mpd_t *a, const mpd_t *b); -int mpd_compare_total_mag(mpd_t *result, const mpd_t *a, const mpd_t *b); - -void mpd_qround_to_intx(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qround_to_int(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qtrunc(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qfloor(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qceil(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); - -void mpd_qabs(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmax(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmax_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmin(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmin_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qminus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qplus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qnext_minus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qnext_plus(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qnext_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qquantize(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qrescale(mpd_t *result, const mpd_t *a, mpd_ssize_t exp, const mpd_context_t *ctx, uint32_t *status); -void mpd_qrescale_fmt(mpd_t *result, const mpd_t *a, mpd_ssize_t exp, const mpd_context_t *ctx, uint32_t *status); -void mpd_qreduce(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qadd(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qadd_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qadd_i32(mpd_t *result, const mpd_t *a, int32_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qadd_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qadd_u32(mpd_t *result, const mpd_t *a, uint32_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsub(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsub_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsub_i32(mpd_t *result, const mpd_t *a, int32_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsub_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsub_u32(mpd_t *result, const mpd_t *a, uint32_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmul(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmul_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmul_i32(mpd_t *result, const mpd_t *a, int32_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmul_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmul_u32(mpd_t *result, const mpd_t *a, uint32_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qfma(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdiv_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdiv_i32(mpd_t *result, const mpd_t *a, int32_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdiv_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdiv_u32(mpd_t *result, const mpd_t *a, uint32_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdivint(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qrem(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qrem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qpow(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_context_t *ctx, uint32_t *status); -void mpd_qpowmod(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_t *mod, const mpd_context_t *ctx, uint32_t *status); -void mpd_qexp(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qln10(mpd_t *result, mpd_ssize_t prec, uint32_t *status); -void mpd_qln(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qlog10(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qinvroot(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status); - -#ifndef LEGACY_COMPILER -void mpd_qadd_i64(mpd_t *result, const mpd_t *a, int64_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qadd_u64(mpd_t *result, const mpd_t *a, uint64_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsub_i64(mpd_t *result, const mpd_t *a, int64_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsub_u64(mpd_t *result, const mpd_t *a, uint64_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmul_i64(mpd_t *result, const mpd_t *a, int64_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qmul_u64(mpd_t *result, const mpd_t *a, uint64_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdiv_i64(mpd_t *result, const mpd_t *a, int64_t b, const mpd_context_t *ctx, uint32_t *status); -void mpd_qdiv_u64(mpd_t *result, const mpd_t *a, uint64_t b, const mpd_context_t *ctx, uint32_t *status); -#endif - - -size_t mpd_sizeinbase(const mpd_t *a, uint32_t base); -void mpd_qimport_u16(mpd_t *result, const uint16_t *srcdata, size_t srclen, - uint8_t srcsign, uint32_t srcbase, - const mpd_context_t *ctx, uint32_t *status); -void mpd_qimport_u32(mpd_t *result, const uint32_t *srcdata, size_t srclen, - uint8_t srcsign, uint32_t srcbase, - const mpd_context_t *ctx, uint32_t *status); -size_t mpd_qexport_u16(uint16_t **rdata, size_t rlen, uint32_t base, - const mpd_t *src, uint32_t *status); -size_t mpd_qexport_u32(uint32_t **rdata, size_t rlen, uint32_t base, - const mpd_t *src, uint32_t *status); - - -/******************************************************************************/ -/* Signalling functions */ -/******************************************************************************/ - -char *mpd_format(const mpd_t *dec, const char *fmt, mpd_context_t *ctx); -void mpd_import_u16(mpd_t *result, const uint16_t *srcdata, size_t srclen, uint8_t srcsign, uint32_t base, mpd_context_t *ctx); -void mpd_import_u32(mpd_t *result, const uint32_t *srcdata, size_t srclen, uint8_t srcsign, uint32_t base, mpd_context_t *ctx); -size_t mpd_export_u16(uint16_t **rdata, size_t rlen, uint32_t base, const mpd_t *src, mpd_context_t *ctx); -size_t mpd_export_u32(uint32_t **rdata, size_t rlen, uint32_t base, const mpd_t *src, mpd_context_t *ctx); -void mpd_finalize(mpd_t *result, mpd_context_t *ctx); -int mpd_check_nan(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -int mpd_check_nans(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_set_string(mpd_t *result, const char *s, mpd_context_t *ctx); -void mpd_maxcoeff(mpd_t *result, mpd_context_t *ctx); -void mpd_sset_ssize(mpd_t *result, mpd_ssize_t a, mpd_context_t *ctx); -void mpd_sset_i32(mpd_t *result, int32_t a, mpd_context_t *ctx); -void mpd_sset_uint(mpd_t *result, mpd_uint_t a, mpd_context_t *ctx); -void mpd_sset_u32(mpd_t *result, uint32_t a, mpd_context_t *ctx); -void mpd_set_ssize(mpd_t *result, mpd_ssize_t a, mpd_context_t *ctx); -void mpd_set_i32(mpd_t *result, int32_t a, mpd_context_t *ctx); -void mpd_set_uint(mpd_t *result, mpd_uint_t a, mpd_context_t *ctx); -void mpd_set_u32(mpd_t *result, uint32_t a, mpd_context_t *ctx); -#ifndef LEGACY_COMPILER -void mpd_set_i64(mpd_t *result, int64_t a, mpd_context_t *ctx); -void mpd_set_u64(mpd_t *result, uint64_t a, mpd_context_t *ctx); -#endif -mpd_ssize_t mpd_get_ssize(const mpd_t *a, mpd_context_t *ctx); -mpd_uint_t mpd_get_uint(const mpd_t *a, mpd_context_t *ctx); -mpd_uint_t mpd_abs_uint(const mpd_t *a, mpd_context_t *ctx); -int32_t mpd_get_i32(const mpd_t *a, mpd_context_t *ctx); -uint32_t mpd_get_u32(const mpd_t *a, mpd_context_t *ctx); -#ifndef LEGACY_COMPILER -int64_t mpd_get_i64(const mpd_t *a, mpd_context_t *ctx); -uint64_t mpd_get_u64(const mpd_t *a, mpd_context_t *ctx); -#endif -void mpd_and(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_copy(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_canonical(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_copy_abs(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_copy_negate(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_copy_sign(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_invert(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_logb(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_or(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_rotate(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_scaleb(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_shiftl(mpd_t *result, const mpd_t *a, mpd_ssize_t n, mpd_context_t *ctx); -mpd_uint_t mpd_shiftr(mpd_t *result, const mpd_t *a, mpd_ssize_t n, mpd_context_t *ctx); -void mpd_shiftn(mpd_t *result, const mpd_t *a, mpd_ssize_t n, mpd_context_t *ctx); -void mpd_shift(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_xor(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_abs(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -int mpd_cmp(const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -int mpd_compare(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -int mpd_compare_signal(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_add(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_add_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, mpd_context_t *ctx); -void mpd_add_i32(mpd_t *result, const mpd_t *a, int32_t b, mpd_context_t *ctx); -void mpd_add_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, mpd_context_t *ctx); -void mpd_add_u32(mpd_t *result, const mpd_t *a, uint32_t b, mpd_context_t *ctx); -void mpd_sub(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_sub_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, mpd_context_t *ctx); -void mpd_sub_i32(mpd_t *result, const mpd_t *a, int32_t b, mpd_context_t *ctx); -void mpd_sub_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, mpd_context_t *ctx); -void mpd_sub_u32(mpd_t *result, const mpd_t *a, uint32_t b, mpd_context_t *ctx); -void mpd_div(mpd_t *q, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_div_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, mpd_context_t *ctx); -void mpd_div_i32(mpd_t *result, const mpd_t *a, int32_t b, mpd_context_t *ctx); -void mpd_div_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, mpd_context_t *ctx); -void mpd_div_u32(mpd_t *result, const mpd_t *a, uint32_t b, mpd_context_t *ctx); -void mpd_divmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_divint(mpd_t *q, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_exp(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_fma(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, mpd_context_t *ctx); -void mpd_ln(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_log10(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_max(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_max_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_min(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_min_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_minus(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_mul(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_mul_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, mpd_context_t *ctx); -void mpd_mul_i32(mpd_t *result, const mpd_t *a, int32_t b, mpd_context_t *ctx); -void mpd_mul_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, mpd_context_t *ctx); -void mpd_mul_u32(mpd_t *result, const mpd_t *a, uint32_t b, mpd_context_t *ctx); -void mpd_next_minus(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_next_plus(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_next_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_plus(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_pow(mpd_t *result, const mpd_t *base, const mpd_t *exp, mpd_context_t *ctx); -void mpd_powmod(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_t *mod, mpd_context_t *ctx); -void mpd_quantize(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_rescale(mpd_t *result, const mpd_t *a, mpd_ssize_t exp, mpd_context_t *ctx); -void mpd_reduce(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_rem(mpd_t *r, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_rem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx); -void mpd_round_to_intx(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_round_to_int(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_trunc(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_floor(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_ceil(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_sqrt(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); -void mpd_invroot(mpd_t *result, const mpd_t *a, mpd_context_t *ctx); - -#ifndef LEGACY_COMPILER -void mpd_add_i64(mpd_t *result, const mpd_t *a, int64_t b, mpd_context_t *ctx); -void mpd_add_u64(mpd_t *result, const mpd_t *a, uint64_t b, mpd_context_t *ctx); -void mpd_sub_i64(mpd_t *result, const mpd_t *a, int64_t b, mpd_context_t *ctx); -void mpd_sub_u64(mpd_t *result, const mpd_t *a, uint64_t b, mpd_context_t *ctx); -void mpd_div_i64(mpd_t *result, const mpd_t *a, int64_t b, mpd_context_t *ctx); -void mpd_div_u64(mpd_t *result, const mpd_t *a, uint64_t b, mpd_context_t *ctx); -void mpd_mul_i64(mpd_t *result, const mpd_t *a, int64_t b, mpd_context_t *ctx); -void mpd_mul_u64(mpd_t *result, const mpd_t *a, uint64_t b, mpd_context_t *ctx); -#endif - - -/******************************************************************************/ -/* Configuration specific */ -/******************************************************************************/ - -#ifdef CONFIG_64 -void mpd_qsset_i64(mpd_t *result, int64_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_qsset_u64(mpd_t *result, uint64_t a, const mpd_context_t *ctx, uint32_t *status); -void mpd_sset_i64(mpd_t *result, int64_t a, mpd_context_t *ctx); -void mpd_sset_u64(mpd_t *result, uint64_t a, mpd_context_t *ctx); -#endif - - -/******************************************************************************/ -/* Get attributes of a decimal */ -/******************************************************************************/ - -EXTINLINE mpd_ssize_t mpd_adjexp(const mpd_t *dec); -EXTINLINE mpd_ssize_t mpd_etiny(const mpd_context_t *ctx); -EXTINLINE mpd_ssize_t mpd_etop(const mpd_context_t *ctx); -EXTINLINE mpd_uint_t mpd_msword(const mpd_t *dec); -EXTINLINE int mpd_word_digits(mpd_uint_t word); -/* most significant digit of a word */ -EXTINLINE mpd_uint_t mpd_msd(mpd_uint_t word); -/* least significant digit of a word */ -EXTINLINE mpd_uint_t mpd_lsd(mpd_uint_t word); -/* coefficient size needed to store 'digits' */ -EXTINLINE mpd_ssize_t mpd_digits_to_size(mpd_ssize_t digits); -/* number of digits in the exponent, undefined for MPD_SSIZE_MIN */ -EXTINLINE int mpd_exp_digits(mpd_ssize_t exp); -EXTINLINE int mpd_iscanonical(const mpd_t *dec); -EXTINLINE int mpd_isfinite(const mpd_t *dec); -EXTINLINE int mpd_isinfinite(const mpd_t *dec); -EXTINLINE int mpd_isinteger(const mpd_t *dec); -EXTINLINE int mpd_isnan(const mpd_t *dec); -EXTINLINE int mpd_isnegative(const mpd_t *dec); -EXTINLINE int mpd_ispositive(const mpd_t *dec); -EXTINLINE int mpd_isqnan(const mpd_t *dec); -EXTINLINE int mpd_issigned(const mpd_t *dec); -EXTINLINE int mpd_issnan(const mpd_t *dec); -EXTINLINE int mpd_isspecial(const mpd_t *dec); -EXTINLINE int mpd_iszero(const mpd_t *dec); -/* undefined for special numbers */ -EXTINLINE int mpd_iszerocoeff(const mpd_t *dec); -EXTINLINE int mpd_isnormal(const mpd_t *dec, const mpd_context_t *ctx); -EXTINLINE int mpd_issubnormal(const mpd_t *dec, const mpd_context_t *ctx); -/* odd word */ -EXTINLINE int mpd_isoddword(mpd_uint_t word); -/* odd coefficient */ -EXTINLINE int mpd_isoddcoeff(const mpd_t *dec); -/* odd decimal, only defined for integers */ -int mpd_isodd(const mpd_t *dec); -/* even decimal, only defined for integers */ -int mpd_iseven(const mpd_t *dec); -/* 0 if dec is positive, 1 if dec is negative */ -EXTINLINE uint8_t mpd_sign(const mpd_t *dec); -/* 1 if dec is positive, -1 if dec is negative */ -EXTINLINE int mpd_arith_sign(const mpd_t *dec); -EXTINLINE long mpd_radix(void); -EXTINLINE int mpd_isdynamic(const mpd_t *dec); -EXTINLINE int mpd_isstatic(const mpd_t *dec); -EXTINLINE int mpd_isdynamic_data(const mpd_t *dec); -EXTINLINE int mpd_isstatic_data(const mpd_t *dec); -EXTINLINE int mpd_isshared_data(const mpd_t *dec); -EXTINLINE int mpd_isconst_data(const mpd_t *dec); -EXTINLINE mpd_ssize_t mpd_trail_zeros(const mpd_t *dec); - - -/******************************************************************************/ -/* Set attributes of a decimal */ -/******************************************************************************/ - -/* set number of decimal digits in the coefficient */ -EXTINLINE void mpd_setdigits(mpd_t *result); -EXTINLINE void mpd_set_sign(mpd_t *result, uint8_t sign); -/* copy sign from another decimal */ -EXTINLINE void mpd_signcpy(mpd_t *result, const mpd_t *a); -EXTINLINE void mpd_set_infinity(mpd_t *result); -EXTINLINE void mpd_set_qnan(mpd_t *result); -EXTINLINE void mpd_set_snan(mpd_t *result); -EXTINLINE void mpd_set_negative(mpd_t *result); -EXTINLINE void mpd_set_positive(mpd_t *result); -EXTINLINE void mpd_set_dynamic(mpd_t *result); -EXTINLINE void mpd_set_static(mpd_t *result); -EXTINLINE void mpd_set_dynamic_data(mpd_t *result); -EXTINLINE void mpd_set_static_data(mpd_t *result); -EXTINLINE void mpd_set_shared_data(mpd_t *result); -EXTINLINE void mpd_set_const_data(mpd_t *result); -EXTINLINE void mpd_clear_flags(mpd_t *result); -EXTINLINE void mpd_set_flags(mpd_t *result, uint8_t flags); -EXTINLINE void mpd_copy_flags(mpd_t *result, const mpd_t *a); - - -/******************************************************************************/ -/* Error Macros */ -/******************************************************************************/ - -#define mpd_err_fatal(...) \ - do {fprintf(stderr, "%s:%d: error: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); fputc('\n', stderr); \ - abort(); \ - } while (0) -#define mpd_err_warn(...) \ - do {fprintf(stderr, "%s:%d: warning: ", __FILE__, __LINE__); \ - fprintf(stderr, __VA_ARGS__); fputc('\n', stderr); \ - } while (0) - - -/******************************************************************************/ -/* Memory handling */ -/******************************************************************************/ - -extern void *(* mpd_mallocfunc)(size_t size); -extern void *(* mpd_callocfunc)(size_t nmemb, size_t size); -extern void *(* mpd_reallocfunc)(void *ptr, size_t size); -extern void (* mpd_free)(void *ptr); - -void *mpd_callocfunc_em(size_t nmemb, size_t size); - -void *mpd_alloc(mpd_size_t nmemb, mpd_size_t size); -void *mpd_calloc(mpd_size_t nmemb, mpd_size_t size); -void *mpd_realloc(void *ptr, mpd_size_t nmemb, mpd_size_t size, uint8_t *err); -void *mpd_sh_alloc(mpd_size_t struct_size, mpd_size_t nmemb, mpd_size_t size); - -mpd_t *mpd_qnew(void); -mpd_t *mpd_new(mpd_context_t *ctx); -mpd_t *mpd_qnew_size(mpd_ssize_t nwords); -EXTINLINE void mpd_del(mpd_t *dec); - -EXTINLINE void mpd_uint_zero(mpd_uint_t *dest, mpd_size_t len); -EXTINLINE int mpd_qresize(mpd_t *result, mpd_ssize_t nwords, uint32_t *status); -EXTINLINE int mpd_qresize_zero(mpd_t *result, mpd_ssize_t nwords, uint32_t *status); -EXTINLINE void mpd_minalloc(mpd_t *result); - -int mpd_resize(mpd_t *result, mpd_ssize_t nwords, mpd_context_t *ctx); -int mpd_resize_zero(mpd_t *result, mpd_ssize_t nwords, mpd_context_t *ctx); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#ifdef __cplusplus -} /* END extern "C" */ -#endif - - -#endif /* LIBMPDEC_MPDECIMAL_H_ */ diff --git a/Modules/_decimal/libmpdec/mpsignal.c b/Modules/_decimal/libmpdec/mpsignal.c deleted file mode 100644 index fc2af48f4f37957..000000000000000 --- a/Modules/_decimal/libmpdec/mpsignal.c +++ /dev/null @@ -1,967 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <stddef.h> -#include <stdint.h> - - -/* Signaling wrappers for the quiet functions in mpdecimal.c. */ - - -char * -mpd_format(const mpd_t *dec, const char *fmt, mpd_context_t *ctx) -{ - char *ret; - uint32_t status = 0; - ret = mpd_qformat(dec, fmt, ctx, &status); - mpd_addstatus_raise(ctx, status); - return ret; -} - -void -mpd_import_u16(mpd_t *result, const uint16_t *srcdata, size_t srclen, - uint8_t srcsign, uint32_t base, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qimport_u16(result, srcdata, srclen, srcsign, base, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_import_u32(mpd_t *result, const uint32_t *srcdata, size_t srclen, - uint8_t srcsign, uint32_t base, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qimport_u32(result, srcdata, srclen, srcsign, base, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -size_t -mpd_export_u16(uint16_t **rdata, size_t rlen, uint32_t base, const mpd_t *src, - mpd_context_t *ctx) -{ - size_t n; - uint32_t status = 0; - n = mpd_qexport_u16(rdata, rlen, base, src, &status); - mpd_addstatus_raise(ctx, status); - return n; -} - -size_t -mpd_export_u32(uint32_t **rdata, size_t rlen, uint32_t base, const mpd_t *src, - mpd_context_t *ctx) -{ - size_t n; - uint32_t status = 0; - n = mpd_qexport_u32(rdata, rlen, base, src, &status); - mpd_addstatus_raise(ctx, status); - return n; -} - -void -mpd_finalize(mpd_t *result, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qfinalize(result, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -int -mpd_check_nan(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - if (mpd_qcheck_nan(result, a, ctx, &status)) { - mpd_addstatus_raise(ctx, status); - return 1; - } - return 0; -} - -int -mpd_check_nans(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - if (mpd_qcheck_nans(result, a, b, ctx, &status)) { - mpd_addstatus_raise(ctx, status); - return 1; - } - return 0; -} - -void -mpd_set_string(mpd_t *result, const char *s, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qset_string(result, s, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_maxcoeff(mpd_t *result, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmaxcoeff(result, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -/* set static mpd from signed integer */ -void -mpd_sset_ssize(mpd_t *result, mpd_ssize_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsset_ssize(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_sset_i32(mpd_t *result, int32_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsset_i32(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifdef CONFIG_64 -void -mpd_sset_i64(mpd_t *result, int64_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsset_i64(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -/* set static mpd from unsigned integer */ -void -mpd_sset_uint(mpd_t *result, mpd_uint_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsset_uint(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_sset_u32(mpd_t *result, uint32_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsset_u32(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifdef CONFIG_64 -void -mpd_sset_u64(mpd_t *result, uint64_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsset_u64(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -/* set mpd from signed integer */ -void -mpd_set_ssize(mpd_t *result, mpd_ssize_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qset_ssize(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_set_i32(mpd_t *result, int32_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qset_i32(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_set_i64(mpd_t *result, int64_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qset_i64(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -/* set mpd from unsigned integer */ -void -mpd_set_uint(mpd_t *result, mpd_uint_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qset_uint(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_set_u32(mpd_t *result, uint32_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qset_u32(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_set_u64(mpd_t *result, uint64_t a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qset_u64(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -/* convert mpd to signed integer */ -mpd_ssize_t -mpd_get_ssize(const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_ssize_t ret; - - ret = mpd_qget_ssize(a, &status); - mpd_addstatus_raise(ctx, status); - return ret; -} - -int32_t -mpd_get_i32(const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - int32_t ret; - - ret = mpd_qget_i32(a, &status); - mpd_addstatus_raise(ctx, status); - return ret; -} - -#ifndef LEGACY_COMPILER -int64_t -mpd_get_i64(const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - int64_t ret; - - ret = mpd_qget_i64(a, &status); - mpd_addstatus_raise(ctx, status); - return ret; -} -#endif - -mpd_uint_t -mpd_get_uint(const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_uint_t ret; - - ret = mpd_qget_uint(a, &status); - mpd_addstatus_raise(ctx, status); - return ret; -} - -mpd_uint_t -mpd_abs_uint(const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_uint_t ret; - - ret = mpd_qabs_uint(a, &status); - mpd_addstatus_raise(ctx, status); - return ret; -} - -uint32_t -mpd_get_u32(const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - uint32_t ret; - - ret = mpd_qget_u32(a, &status); - mpd_addstatus_raise(ctx, status); - return ret; -} - -#ifndef LEGACY_COMPILER -uint64_t -mpd_get_u64(const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - uint64_t ret; - - ret = mpd_qget_u64(a, &status); - mpd_addstatus_raise(ctx, status); - return ret; -} -#endif - -void -mpd_and(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qand(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_copy(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - if (!mpd_qcopy(result, a, &status)) { - mpd_addstatus_raise(ctx, status); - } -} - -void -mpd_canonical(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - mpd_copy(result, a, ctx); -} - -void -mpd_copy_abs(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - if (!mpd_qcopy_abs(result, a, &status)) { - mpd_addstatus_raise(ctx, status); - } -} - -void -mpd_copy_negate(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - if (!mpd_qcopy_negate(result, a, &status)) { - mpd_addstatus_raise(ctx, status); - } -} - -void -mpd_copy_sign(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - if (!mpd_qcopy_sign(result, a, b, &status)) { - mpd_addstatus_raise(ctx, status); - } -} - -void -mpd_invert(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qinvert(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_logb(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qlogb(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_or(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qor(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_rotate(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qrotate(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_scaleb(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qscaleb(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_shiftl(mpd_t *result, const mpd_t *a, mpd_ssize_t n, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qshiftl(result, a, n, &status); - mpd_addstatus_raise(ctx, status); -} - -mpd_uint_t -mpd_shiftr(mpd_t *result, const mpd_t *a, mpd_ssize_t n, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_uint_t rnd; - - rnd = mpd_qshiftr(result, a, n, &status); - mpd_addstatus_raise(ctx, status); - return rnd; -} - -void -mpd_shiftn(mpd_t *result, const mpd_t *a, mpd_ssize_t n, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qshiftn(result, a, n, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_shift(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qshift(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_xor(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qxor(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_abs(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qabs(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -int -mpd_cmp(const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - int c; - c = mpd_qcmp(a, b, &status); - mpd_addstatus_raise(ctx, status); - return c; -} - -int -mpd_compare(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - int c; - c = mpd_qcompare(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); - return c; -} - -int -mpd_compare_signal(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - int c; - c = mpd_qcompare_signal(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); - return c; -} - -void -mpd_add(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qadd(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_sub(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsub(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_add_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qadd_ssize(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_add_i32(mpd_t *result, const mpd_t *a, int32_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qadd_i32(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_add_i64(mpd_t *result, const mpd_t *a, int64_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qadd_i64(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -void -mpd_add_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qadd_uint(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_add_u32(mpd_t *result, const mpd_t *a, uint32_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qadd_u32(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_add_u64(mpd_t *result, const mpd_t *a, uint64_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qadd_u64(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -void -mpd_sub_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsub_ssize(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_sub_i32(mpd_t *result, const mpd_t *a, int32_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsub_i32(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_sub_i64(mpd_t *result, const mpd_t *a, int64_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsub_i64(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -void -mpd_sub_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsub_uint(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_sub_u32(mpd_t *result, const mpd_t *a, uint32_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsub_u32(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_sub_u64(mpd_t *result, const mpd_t *a, uint64_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsub_u64(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -void -mpd_div(mpd_t *q, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdiv(q, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_div_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdiv_ssize(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_div_i32(mpd_t *result, const mpd_t *a, int32_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdiv_i32(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_div_i64(mpd_t *result, const mpd_t *a, int64_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdiv_i64(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -void -mpd_div_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdiv_uint(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_div_u32(mpd_t *result, const mpd_t *a, uint32_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdiv_u32(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_div_u64(mpd_t *result, const mpd_t *a, uint64_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdiv_u64(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -void -mpd_divmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdivmod(q, r, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_divint(mpd_t *q, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qdivint(q, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_exp(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qexp(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_fma(mpd_t *result, const mpd_t *a, const mpd_t *b, const mpd_t *c, - mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qfma(result, a, b, c, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_ln(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qln(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_log10(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qlog10(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_max(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmax(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_max_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmax_mag(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_min(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmin(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_min_mag(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmin_mag(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_minus(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qminus(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_mul(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmul(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_mul_ssize(mpd_t *result, const mpd_t *a, mpd_ssize_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmul_ssize(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_mul_i32(mpd_t *result, const mpd_t *a, int32_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmul_i32(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_mul_i64(mpd_t *result, const mpd_t *a, int64_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmul_i64(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -void -mpd_mul_uint(mpd_t *result, const mpd_t *a, mpd_uint_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmul_uint(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_mul_u32(mpd_t *result, const mpd_t *a, uint32_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmul_u32(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -#ifndef LEGACY_COMPILER -void -mpd_mul_u64(mpd_t *result, const mpd_t *a, uint64_t b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qmul_u64(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} -#endif - -void -mpd_next_minus(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qnext_minus(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_next_plus(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qnext_plus(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_next_toward(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qnext_toward(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_plus(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qplus(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_pow(mpd_t *result, const mpd_t *base, const mpd_t *exp, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qpow(result, base, exp, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_powmod(mpd_t *result, const mpd_t *base, const mpd_t *exp, const mpd_t *mod, - mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qpowmod(result, base, exp, mod, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_quantize(mpd_t *result, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qquantize(result, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_rescale(mpd_t *result, const mpd_t *a, mpd_ssize_t exp, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qrescale(result, a, exp, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_reduce(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qreduce(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_rem(mpd_t *r, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qrem(r, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_rem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qrem_near(r, a, b, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_round_to_intx(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qround_to_intx(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_round_to_int(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qround_to_int(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_trunc(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qtrunc(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_floor(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qfloor(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_ceil(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qceil(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_sqrt(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qsqrt(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} - -void -mpd_invroot(mpd_t *result, const mpd_t *a, mpd_context_t *ctx) -{ - uint32_t status = 0; - mpd_qinvroot(result, a, ctx, &status); - mpd_addstatus_raise(ctx, status); -} diff --git a/Modules/_decimal/libmpdec/numbertheory.c b/Modules/_decimal/libmpdec/numbertheory.c deleted file mode 100644 index 210e0deb3712034..000000000000000 --- a/Modules/_decimal/libmpdec/numbertheory.c +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> -#include <stdlib.h> - -#include "bits.h" -#include "numbertheory.h" -#include "umodarith.h" - - -/* Bignum: Initialize the Number Theoretic Transform. */ - - -/* - * Return the nth root of unity in F(p). This corresponds to e**((2*pi*i)/n) - * in the Fourier transform. We have w**n == 1 (mod p). - * n := transform length. - * sign := -1 for forward transform, 1 for backward transform. - * modnum := one of {P1, P2, P3}. - */ -mpd_uint_t -_mpd_getkernel(mpd_uint_t n, int sign, int modnum) -{ - mpd_uint_t umod, p, r, xi; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - - SETMODULUS(modnum); - r = mpd_roots[modnum]; /* primitive root of F(p) */ - p = umod; - xi = (p-1) / n; - - if (sign == -1) - return POWMOD(r, (p-1-xi)); - else - return POWMOD(r, xi); -} - -/* - * Initialize and return transform parameters. - * n := transform length. - * sign := -1 for forward transform, 1 for backward transform. - * modnum := one of {P1, P2, P3}. - */ -struct fnt_params * -_mpd_init_fnt_params(mpd_size_t n, int sign, int modnum) -{ - struct fnt_params *tparams; - mpd_uint_t umod; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_uint_t kernel, w; - mpd_uint_t i; - mpd_size_t nhalf; - - assert(ispower2(n)); - assert(sign == -1 || sign == 1); - assert(P1 <= modnum && modnum <= P3); - - nhalf = n/2; - tparams = mpd_sh_alloc(sizeof *tparams, nhalf, sizeof (mpd_uint_t)); - if (tparams == NULL) { - return NULL; - } - - SETMODULUS(modnum); - kernel = _mpd_getkernel(n, sign, modnum); - - tparams->modnum = modnum; - tparams->modulus = umod; - tparams->kernel = kernel; - - /* wtable[] := w**0, w**1, ..., w**(nhalf-1) */ - w = 1; - for (i = 0; i < nhalf; i++) { - tparams->wtable[i] = w; - w = MULMOD(w, kernel); - } - - return tparams; -} - -/* Initialize wtable of size three. */ -void -_mpd_init_w3table(mpd_uint_t w3table[3], int sign, int modnum) -{ - mpd_uint_t umod; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_uint_t kernel; - - SETMODULUS(modnum); - kernel = _mpd_getkernel(3, sign, modnum); - - w3table[0] = 1; - w3table[1] = kernel; - w3table[2] = POWMOD(kernel, 2); -} diff --git a/Modules/_decimal/libmpdec/numbertheory.h b/Modules/_decimal/libmpdec/numbertheory.h deleted file mode 100644 index 47b7753b831b89b..000000000000000 --- a/Modules/_decimal/libmpdec/numbertheory.h +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_NUMBERTHEORY_H_ -#define LIBMPDEC_NUMBERTHEORY_H_ - - -#include "mpdecimal.h" -#include "constants.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -/* transform parameters */ -struct fnt_params { - int modnum; - mpd_uint_t modulus; - mpd_uint_t kernel; - mpd_uint_t wtable[]; -}; - - -mpd_uint_t _mpd_getkernel(mpd_uint_t n, int sign, int modnum); -struct fnt_params *_mpd_init_fnt_params(mpd_size_t n, int sign, int modnum); -void _mpd_init_w3table(mpd_uint_t w3table[3], int sign, int modnum); - - -#ifdef PPRO -static inline void -ppro_setmodulus(int modnum, mpd_uint_t *umod, double *dmod, uint32_t dinvmod[3]) -{ - *dmod = *umod = mpd_moduli[modnum]; - dinvmod[0] = mpd_invmoduli[modnum][0]; - dinvmod[1] = mpd_invmoduli[modnum][1]; - dinvmod[2] = mpd_invmoduli[modnum][2]; -} -#else -static inline void -std_setmodulus(int modnum, mpd_uint_t *umod) -{ - *umod = mpd_moduli[modnum]; -} -#endif - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_NUMBERTHEORY_H_ */ diff --git a/Modules/_decimal/libmpdec/sixstep.c b/Modules/_decimal/libmpdec/sixstep.c deleted file mode 100644 index a4d1dbed7813c6c..000000000000000 --- a/Modules/_decimal/libmpdec/sixstep.c +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> -#include <stdio.h> - -#include "bits.h" -#include "constants.h" -#include "difradix2.h" -#include "numbertheory.h" -#include "sixstep.h" -#include "transpose.h" -#include "umodarith.h" - - -/* Bignum: Cache efficient Matrix Fourier Transform for arrays of the - form 2**n (See literature/six-step.txt). */ - - -/* forward transform with sign = -1 */ -int -six_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) -{ - struct fnt_params *tparams; - mpd_size_t log2n, C, R; - mpd_uint_t kernel; - mpd_uint_t umod; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_uint_t *x, w0, w1, wstep; - mpd_size_t i, k; - - - assert(ispower2(n)); - assert(n >= 16); - assert(n <= MPD_MAXTRANSFORM_2N); - - log2n = mpd_bsr(n); - C = ((mpd_size_t)1) << (log2n / 2); /* number of columns */ - R = ((mpd_size_t)1) << (log2n - (log2n / 2)); /* number of rows */ - - - /* Transpose the matrix. */ - if (!transpose_pow2(a, R, C)) { - return 0; - } - - /* Length R transform on the rows. */ - if ((tparams = _mpd_init_fnt_params(R, -1, modnum)) == NULL) { - return 0; - } - for (x = a; x < a+n; x += R) { - fnt_dif2(x, R, tparams); - } - - /* Transpose the matrix. */ - if (!transpose_pow2(a, C, R)) { - mpd_free(tparams); - return 0; - } - - /* Multiply each matrix element (addressed by i*C+k) by r**(i*k). */ - SETMODULUS(modnum); - kernel = _mpd_getkernel(n, -1, modnum); - for (i = 1; i < R; i++) { - w0 = 1; /* r**(i*0): initial value for k=0 */ - w1 = POWMOD(kernel, i); /* r**(i*1): initial value for k=1 */ - wstep = MULMOD(w1, w1); /* r**(2*i) */ - for (k = 0; k < C; k += 2) { - mpd_uint_t x0 = a[i*C+k]; - mpd_uint_t x1 = a[i*C+k+1]; - MULMOD2(&x0, w0, &x1, w1); - MULMOD2C(&w0, &w1, wstep); /* r**(i*(k+2)) = r**(i*k) * r**(2*i) */ - a[i*C+k] = x0; - a[i*C+k+1] = x1; - } - } - - /* Length C transform on the rows. */ - if (C != R) { - mpd_free(tparams); - if ((tparams = _mpd_init_fnt_params(C, -1, modnum)) == NULL) { - return 0; - } - } - for (x = a; x < a+n; x += C) { - fnt_dif2(x, C, tparams); - } - mpd_free(tparams); - -#if 0 - /* An unordered transform is sufficient for convolution. */ - /* Transpose the matrix. */ - if (!transpose_pow2(a, R, C)) { - return 0; - } -#endif - - return 1; -} - - -/* reverse transform, sign = 1 */ -int -inv_six_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) -{ - struct fnt_params *tparams; - mpd_size_t log2n, C, R; - mpd_uint_t kernel; - mpd_uint_t umod; -#ifdef PPRO - double dmod; - uint32_t dinvmod[3]; -#endif - mpd_uint_t *x, w0, w1, wstep; - mpd_size_t i, k; - - - assert(ispower2(n)); - assert(n >= 16); - assert(n <= MPD_MAXTRANSFORM_2N); - - log2n = mpd_bsr(n); - C = ((mpd_size_t)1) << (log2n / 2); /* number of columns */ - R = ((mpd_size_t)1) << (log2n - (log2n / 2)); /* number of rows */ - - -#if 0 - /* An unordered transform is sufficient for convolution. */ - /* Transpose the matrix, producing an R*C matrix. */ - if (!transpose_pow2(a, C, R)) { - return 0; - } -#endif - - /* Length C transform on the rows. */ - if ((tparams = _mpd_init_fnt_params(C, 1, modnum)) == NULL) { - return 0; - } - for (x = a; x < a+n; x += C) { - fnt_dif2(x, C, tparams); - } - - /* Multiply each matrix element (addressed by i*C+k) by r**(i*k). */ - SETMODULUS(modnum); - kernel = _mpd_getkernel(n, 1, modnum); - for (i = 1; i < R; i++) { - w0 = 1; - w1 = POWMOD(kernel, i); - wstep = MULMOD(w1, w1); - for (k = 0; k < C; k += 2) { - mpd_uint_t x0 = a[i*C+k]; - mpd_uint_t x1 = a[i*C+k+1]; - MULMOD2(&x0, w0, &x1, w1); - MULMOD2C(&w0, &w1, wstep); - a[i*C+k] = x0; - a[i*C+k+1] = x1; - } - } - - /* Transpose the matrix. */ - if (!transpose_pow2(a, R, C)) { - mpd_free(tparams); - return 0; - } - - /* Length R transform on the rows. */ - if (R != C) { - mpd_free(tparams); - if ((tparams = _mpd_init_fnt_params(R, 1, modnum)) == NULL) { - return 0; - } - } - for (x = a; x < a+n; x += R) { - fnt_dif2(x, R, tparams); - } - mpd_free(tparams); - - /* Transpose the matrix. */ - if (!transpose_pow2(a, C, R)) { - return 0; - } - - return 1; -} diff --git a/Modules/_decimal/libmpdec/sixstep.h b/Modules/_decimal/libmpdec/sixstep.h deleted file mode 100644 index 89b4a33afc79207..000000000000000 --- a/Modules/_decimal/libmpdec/sixstep.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_SIXSTEP_H_ -#define LIBMPDEC_SIXSTEP_H_ - - -#include "mpdecimal.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -int six_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum); -int inv_six_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum); - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_SIXSTEP_H_ */ diff --git a/Modules/_decimal/libmpdec/transpose.c b/Modules/_decimal/libmpdec/transpose.c deleted file mode 100644 index 56321b5f39a733e..000000000000000 --- a/Modules/_decimal/libmpdec/transpose.c +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#include "mpdecimal.h" - -#include <assert.h> -#include <limits.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include "bits.h" -#include "constants.h" -#include "transpose.h" -#include "typearith.h" - - -#define BUFSIZE 4096 -#define SIDE 128 - - -/* Bignum: The transpose functions are used for very large transforms - in sixstep.c and fourstep.c. */ - - -/* Definition of the matrix transpose */ -void -std_trans(mpd_uint_t dest[], mpd_uint_t src[], mpd_size_t rows, mpd_size_t cols) -{ - mpd_size_t idest, isrc; - mpd_size_t r, c; - - for (r = 0; r < rows; r++) { - isrc = r * cols; - idest = r; - for (c = 0; c < cols; c++) { - dest[idest] = src[isrc]; - isrc += 1; - idest += rows; - } - } -} - -/* - * Swap half-rows of 2^n * (2*2^n) matrix. - * FORWARD_CYCLE: even/odd permutation of the halfrows. - * BACKWARD_CYCLE: reverse the even/odd permutation. - */ -static int -swap_halfrows_pow2(mpd_uint_t *matrix, mpd_size_t rows, mpd_size_t cols, int dir) -{ - mpd_uint_t buf1[BUFSIZE]; - mpd_uint_t buf2[BUFSIZE]; - mpd_uint_t *readbuf, *writebuf, *hp; - mpd_size_t *done, dbits; - mpd_size_t b = BUFSIZE, stride; - mpd_size_t hn, hmax; /* halfrow number */ - mpd_size_t m, r=0; - mpd_size_t offset; - mpd_size_t next; - - - assert(cols == mul_size_t(2, rows)); - - if (dir == FORWARD_CYCLE) { - r = rows; - } - else if (dir == BACKWARD_CYCLE) { - r = 2; - } - else { - abort(); /* GCOV_NOT_REACHED */ - } - - m = cols - 1; - hmax = rows; /* cycles start at odd halfrows */ - dbits = 8 * sizeof *done; - if ((done = mpd_calloc(hmax/(sizeof *done) + 1, sizeof *done)) == NULL) { - return 0; - } - - for (hn = 1; hn <= hmax; hn += 2) { - - if (done[hn/dbits] & mpd_bits[hn%dbits]) { - continue; - } - - readbuf = buf1; writebuf = buf2; - - for (offset = 0; offset < cols/2; offset += b) { - - stride = (offset + b < cols/2) ? b : cols/2-offset; - - hp = matrix + hn*cols/2; - memcpy(readbuf, hp+offset, stride*(sizeof *readbuf)); - pointerswap(&readbuf, &writebuf); - - next = mulmod_size_t(hn, r, m); - hp = matrix + next*cols/2; - - while (next != hn) { - - memcpy(readbuf, hp+offset, stride*(sizeof *readbuf)); - memcpy(hp+offset, writebuf, stride*(sizeof *writebuf)); - pointerswap(&readbuf, &writebuf); - - done[next/dbits] |= mpd_bits[next%dbits]; - - next = mulmod_size_t(next, r, m); - hp = matrix + next*cols/2; - - } - - memcpy(hp+offset, writebuf, stride*(sizeof *writebuf)); - - done[hn/dbits] |= mpd_bits[hn%dbits]; - } - } - - mpd_free(done); - return 1; -} - -/* In-place transpose of a square matrix */ -static inline void -squaretrans(mpd_uint_t *buf, mpd_size_t cols) -{ - mpd_uint_t tmp; - mpd_size_t idest, isrc; - mpd_size_t r, c; - - for (r = 0; r < cols; r++) { - c = r+1; - isrc = r*cols + c; - idest = c*cols + r; - for (c = r+1; c < cols; c++) { - tmp = buf[isrc]; - buf[isrc] = buf[idest]; - buf[idest] = tmp; - isrc += 1; - idest += cols; - } - } -} - -/* - * Transpose 2^n * 2^n matrix. For cache efficiency, the matrix is split into - * square blocks with side length 'SIDE'. First, the blocks are transposed, - * then a square transposition is done on each individual block. - */ -static void -squaretrans_pow2(mpd_uint_t *matrix, mpd_size_t size) -{ - mpd_uint_t buf1[SIDE*SIDE]; - mpd_uint_t buf2[SIDE*SIDE]; - mpd_uint_t *to, *from; - mpd_size_t b = size; - mpd_size_t r, c; - mpd_size_t i; - - while (b > SIDE) b >>= 1; - - for (r = 0; r < size; r += b) { - - for (c = r; c < size; c += b) { - - from = matrix + r*size + c; - to = buf1; - for (i = 0; i < b; i++) { - memcpy(to, from, b*(sizeof *to)); - from += size; - to += b; - } - squaretrans(buf1, b); - - if (r == c) { - to = matrix + r*size + c; - from = buf1; - for (i = 0; i < b; i++) { - memcpy(to, from, b*(sizeof *to)); - from += b; - to += size; - } - continue; - } - else { - from = matrix + c*size + r; - to = buf2; - for (i = 0; i < b; i++) { - memcpy(to, from, b*(sizeof *to)); - from += size; - to += b; - } - squaretrans(buf2, b); - - to = matrix + c*size + r; - from = buf1; - for (i = 0; i < b; i++) { - memcpy(to, from, b*(sizeof *to)); - from += b; - to += size; - } - - to = matrix + r*size + c; - from = buf2; - for (i = 0; i < b; i++) { - memcpy(to, from, b*(sizeof *to)); - from += b; - to += size; - } - } - } - } - -} - -/* - * In-place transposition of a 2^n x 2^n or a 2^n x (2*2^n) - * or a (2*2^n) x 2^n matrix. - */ -int -transpose_pow2(mpd_uint_t *matrix, mpd_size_t rows, mpd_size_t cols) -{ - mpd_size_t size = mul_size_t(rows, cols); - - assert(ispower2(rows)); - assert(ispower2(cols)); - - if (cols == rows) { - squaretrans_pow2(matrix, rows); - } - else if (cols == mul_size_t(2, rows)) { - if (!swap_halfrows_pow2(matrix, rows, cols, FORWARD_CYCLE)) { - return 0; - } - squaretrans_pow2(matrix, rows); - squaretrans_pow2(matrix+(size/2), rows); - } - else if (rows == mul_size_t(2, cols)) { - squaretrans_pow2(matrix, cols); - squaretrans_pow2(matrix+(size/2), cols); - if (!swap_halfrows_pow2(matrix, cols, rows, BACKWARD_CYCLE)) { - return 0; - } - } - else { - abort(); /* GCOV_NOT_REACHED */ - } - - return 1; -} diff --git a/Modules/_decimal/libmpdec/transpose.h b/Modules/_decimal/libmpdec/transpose.h deleted file mode 100644 index e91c18d74356bcf..000000000000000 --- a/Modules/_decimal/libmpdec/transpose.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_TRANSPOSE_H_ -#define LIBMPDEC_TRANSPOSE_H_ - - -#include "mpdecimal.h" - - -/* Internal header file: all symbols have local scope in the DSO */ -MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) - - -enum {FORWARD_CYCLE, BACKWARD_CYCLE}; - - -void std_trans(mpd_uint_t dest[], mpd_uint_t src[], mpd_size_t rows, mpd_size_t cols); -int transpose_pow2(mpd_uint_t *matrix, mpd_size_t rows, mpd_size_t cols); -void transpose_3xpow2(mpd_uint_t *matrix, mpd_size_t rows, mpd_size_t cols); - - -static inline void pointerswap(mpd_uint_t **a, mpd_uint_t **b) -{ - mpd_uint_t *tmp; - - tmp = *b; - *b = *a; - *a = tmp; -} - - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ - - -#endif /* LIBMPDEC_TRANSPOSE_H_ */ diff --git a/Modules/_decimal/libmpdec/typearith.h b/Modules/_decimal/libmpdec/typearith.h deleted file mode 100644 index dd3776453d098d5..000000000000000 --- a/Modules/_decimal/libmpdec/typearith.h +++ /dev/null @@ -1,668 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_TYPEARITH_H_ -#define LIBMPDEC_TYPEARITH_H_ - - -#include "mpdecimal.h" - -#include <assert.h> - - -/*****************************************************************************/ -/* Low level native arithmetic on basic types */ -/*****************************************************************************/ - - -/** ------------------------------------------------------------ - ** Double width multiplication and division - ** ------------------------------------------------------------ - */ - -#if defined(CONFIG_64) -#if defined(ANSI) -#if defined(HAVE_UINT128_T) -static inline void -_mpd_mul_words(mpd_uint_t *hi, mpd_uint_t *lo, mpd_uint_t a, mpd_uint_t b) -{ - __uint128_t hl; - - hl = (__uint128_t)a * b; - - *hi = hl >> 64; - *lo = (mpd_uint_t)hl; -} - -static inline void -_mpd_div_words(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t hi, mpd_uint_t lo, - mpd_uint_t d) -{ - __uint128_t hl; - - hl = ((__uint128_t)hi<<64) + lo; - *q = (mpd_uint_t)(hl / d); /* quotient is known to fit */ - *r = (mpd_uint_t)(hl - (__uint128_t)(*q) * d); -} -#else -static inline void -_mpd_mul_words(mpd_uint_t *hi, mpd_uint_t *lo, mpd_uint_t a, mpd_uint_t b) -{ - uint32_t w[4], carry; - uint32_t ah, al, bh, bl; - uint64_t hl; - - ah = (uint32_t)(a>>32); al = (uint32_t)a; - bh = (uint32_t)(b>>32); bl = (uint32_t)b; - - hl = (uint64_t)al * bl; - w[0] = (uint32_t)hl; - carry = (uint32_t)(hl>>32); - - hl = (uint64_t)ah * bl + carry; - w[1] = (uint32_t)hl; - w[2] = (uint32_t)(hl>>32); - - hl = (uint64_t)al * bh + w[1]; - w[1] = (uint32_t)hl; - carry = (uint32_t)(hl>>32); - - hl = ((uint64_t)ah * bh + w[2]) + carry; - w[2] = (uint32_t)hl; - w[3] = (uint32_t)(hl>>32); - - *hi = ((uint64_t)w[3]<<32) + w[2]; - *lo = ((uint64_t)w[1]<<32) + w[0]; -} - -/* - * By Henry S. Warren: http://www.hackersdelight.org/HDcode/divlu.c.txt - * http://www.hackersdelight.org/permissions.htm: - * "You are free to use, copy, and distribute any of the code on this web - * site, whether modified by you or not. You need not give attribution." - * - * Slightly modified, comments are mine. - */ -static inline int -nlz(uint64_t x) -{ - int n; - - if (x == 0) return(64); - - n = 0; - if (x <= 0x00000000FFFFFFFF) {n = n +32; x = x <<32;} - if (x <= 0x0000FFFFFFFFFFFF) {n = n +16; x = x <<16;} - if (x <= 0x00FFFFFFFFFFFFFF) {n = n + 8; x = x << 8;} - if (x <= 0x0FFFFFFFFFFFFFFF) {n = n + 4; x = x << 4;} - if (x <= 0x3FFFFFFFFFFFFFFF) {n = n + 2; x = x << 2;} - if (x <= 0x7FFFFFFFFFFFFFFF) {n = n + 1;} - - return n; -} - -static inline void -_mpd_div_words(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t u1, mpd_uint_t u0, - mpd_uint_t v) -{ - const mpd_uint_t b = 4294967296; - mpd_uint_t un1, un0, - vn1, vn0, - q1, q0, - un32, un21, un10, - rhat, t; - int s; - - assert(u1 < v); - - s = nlz(v); - v = v << s; - vn1 = v >> 32; - vn0 = v & 0xFFFFFFFF; - - t = (s == 0) ? 0 : u0 >> (64 - s); - un32 = (u1 << s) | t; - un10 = u0 << s; - - un1 = un10 >> 32; - un0 = un10 & 0xFFFFFFFF; - - q1 = un32 / vn1; - rhat = un32 - q1*vn1; -again1: - if (q1 >= b || q1*vn0 > b*rhat + un1) { - q1 = q1 - 1; - rhat = rhat + vn1; - if (rhat < b) goto again1; - } - - /* - * Before again1 we had: - * (1) q1*vn1 + rhat = un32 - * (2) q1*vn1*b + rhat*b + un1 = un32*b + un1 - * - * The statements inside the if-clause do not change the value - * of the left-hand side of (2), and the loop is only exited - * if q1*vn0 <= rhat*b + un1, so: - * - * (3) q1*vn1*b + q1*vn0 <= un32*b + un1 - * (4) q1*v <= un32*b + un1 - * (5) 0 <= un32*b + un1 - q1*v - * - * By (5) we are certain that the possible add-back step from - * Knuth's algorithm D is never required. - * - * Since the final quotient is less than 2**64, the following - * must be true: - * - * (6) un32*b + un1 - q1*v <= UINT64_MAX - * - * This means that in the following line, the high words - * of un32*b and q1*v can be discarded without any effect - * on the result. - */ - un21 = un32*b + un1 - q1*v; - - q0 = un21 / vn1; - rhat = un21 - q0*vn1; -again2: - if (q0 >= b || q0*vn0 > b*rhat + un0) { - q0 = q0 - 1; - rhat = rhat + vn1; - if (rhat < b) goto again2; - } - - *q = q1*b + q0; - *r = (un21*b + un0 - q0*v) >> s; -} -#endif - -/* END ANSI */ -#elif defined(ASM) -static inline void -_mpd_mul_words(mpd_uint_t *hi, mpd_uint_t *lo, mpd_uint_t a, mpd_uint_t b) -{ - mpd_uint_t h, l; - - __asm__ ( "mulq %3\n\t" - : "=d" (h), "=a" (l) - : "%a" (a), "rm" (b) - : "cc" - ); - - *hi = h; - *lo = l; -} - -static inline void -_mpd_div_words(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t hi, mpd_uint_t lo, - mpd_uint_t d) -{ - mpd_uint_t qq, rr; - - __asm__ ( "divq %4\n\t" - : "=a" (qq), "=d" (rr) - : "a" (lo), "d" (hi), "rm" (d) - : "cc" - ); - - *q = qq; - *r = rr; -} -/* END GCC ASM */ -#elif defined(MASM) -#include <intrin.h> -#pragma intrinsic(_umul128) - -static inline void -_mpd_mul_words(mpd_uint_t *hi, mpd_uint_t *lo, mpd_uint_t a, mpd_uint_t b) -{ - *lo = _umul128(a, b, hi); -} - -void _mpd_div_words(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t hi, mpd_uint_t lo, - mpd_uint_t d); - -/* END MASM (_MSC_VER) */ -#else - #error "need platform specific 128 bit multiplication and division" -#endif - -#define DIVMOD(q, r, v, d) *q = v / d; *r = v - *q * d -static inline void -_mpd_divmod_pow10(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t v, mpd_uint_t exp) -{ - assert(exp <= 19); - - if (exp <= 9) { - if (exp <= 4) { - switch (exp) { - case 0: *q = v; *r = 0; break; - case 1: DIVMOD(q, r, v, 10UL); break; - case 2: DIVMOD(q, r, v, 100UL); break; - case 3: DIVMOD(q, r, v, 1000UL); break; - case 4: DIVMOD(q, r, v, 10000UL); break; - } - } - else { - switch (exp) { - case 5: DIVMOD(q, r, v, 100000UL); break; - case 6: DIVMOD(q, r, v, 1000000UL); break; - case 7: DIVMOD(q, r, v, 10000000UL); break; - case 8: DIVMOD(q, r, v, 100000000UL); break; - case 9: DIVMOD(q, r, v, 1000000000UL); break; - } - } - } - else { - if (exp <= 14) { - switch (exp) { - case 10: DIVMOD(q, r, v, 10000000000ULL); break; - case 11: DIVMOD(q, r, v, 100000000000ULL); break; - case 12: DIVMOD(q, r, v, 1000000000000ULL); break; - case 13: DIVMOD(q, r, v, 10000000000000ULL); break; - case 14: DIVMOD(q, r, v, 100000000000000ULL); break; - } - } - else { - switch (exp) { - case 15: DIVMOD(q, r, v, 1000000000000000ULL); break; - case 16: DIVMOD(q, r, v, 10000000000000000ULL); break; - case 17: DIVMOD(q, r, v, 100000000000000000ULL); break; - case 18: DIVMOD(q, r, v, 1000000000000000000ULL); break; - case 19: DIVMOD(q, r, v, 10000000000000000000ULL); break; /* GCOV_NOT_REACHED */ - } - } - } -} - -/* END CONFIG_64 */ -#elif defined(CONFIG_32) -#if defined(ANSI) -#if !defined(LEGACY_COMPILER) -static inline void -_mpd_mul_words(mpd_uint_t *hi, mpd_uint_t *lo, mpd_uint_t a, mpd_uint_t b) -{ - mpd_uuint_t hl; - - hl = (mpd_uuint_t)a * b; - - *hi = hl >> 32; - *lo = (mpd_uint_t)hl; -} - -static inline void -_mpd_div_words(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t hi, mpd_uint_t lo, - mpd_uint_t d) -{ - mpd_uuint_t hl; - - hl = ((mpd_uuint_t)hi<<32) + lo; - *q = (mpd_uint_t)(hl / d); /* quotient is known to fit */ - *r = (mpd_uint_t)(hl - (mpd_uuint_t)(*q) * d); -} -/* END ANSI + uint64_t */ -#else -static inline void -_mpd_mul_words(mpd_uint_t *hi, mpd_uint_t *lo, mpd_uint_t a, mpd_uint_t b) -{ - uint16_t w[4], carry; - uint16_t ah, al, bh, bl; - uint32_t hl; - - ah = (uint16_t)(a>>16); al = (uint16_t)a; - bh = (uint16_t)(b>>16); bl = (uint16_t)b; - - hl = (uint32_t)al * bl; - w[0] = (uint16_t)hl; - carry = (uint16_t)(hl>>16); - - hl = (uint32_t)ah * bl + carry; - w[1] = (uint16_t)hl; - w[2] = (uint16_t)(hl>>16); - - hl = (uint32_t)al * bh + w[1]; - w[1] = (uint16_t)hl; - carry = (uint16_t)(hl>>16); - - hl = ((uint32_t)ah * bh + w[2]) + carry; - w[2] = (uint16_t)hl; - w[3] = (uint16_t)(hl>>16); - - *hi = ((uint32_t)w[3]<<16) + w[2]; - *lo = ((uint32_t)w[1]<<16) + w[0]; -} - -/* - * By Henry S. Warren: http://www.hackersdelight.org/HDcode/divlu.c.txt - * http://www.hackersdelight.org/permissions.htm: - * "You are free to use, copy, and distribute any of the code on this web - * site, whether modified by you or not. You need not give attribution." - * - * Slightly modified, comments are mine. - */ -static inline int -nlz(uint32_t x) -{ - int n; - - if (x == 0) return(32); - - n = 0; - if (x <= 0x0000FFFF) {n = n +16; x = x <<16;} - if (x <= 0x00FFFFFF) {n = n + 8; x = x << 8;} - if (x <= 0x0FFFFFFF) {n = n + 4; x = x << 4;} - if (x <= 0x3FFFFFFF) {n = n + 2; x = x << 2;} - if (x <= 0x7FFFFFFF) {n = n + 1;} - - return n; -} - -static inline void -_mpd_div_words(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t u1, mpd_uint_t u0, - mpd_uint_t v) -{ - const mpd_uint_t b = 65536; - mpd_uint_t un1, un0, - vn1, vn0, - q1, q0, - un32, un21, un10, - rhat, t; - int s; - - assert(u1 < v); - - s = nlz(v); - v = v << s; - vn1 = v >> 16; - vn0 = v & 0xFFFF; - - t = (s == 0) ? 0 : u0 >> (32 - s); - un32 = (u1 << s) | t; - un10 = u0 << s; - - un1 = un10 >> 16; - un0 = un10 & 0xFFFF; - - q1 = un32 / vn1; - rhat = un32 - q1*vn1; -again1: - if (q1 >= b || q1*vn0 > b*rhat + un1) { - q1 = q1 - 1; - rhat = rhat + vn1; - if (rhat < b) goto again1; - } - - /* - * Before again1 we had: - * (1) q1*vn1 + rhat = un32 - * (2) q1*vn1*b + rhat*b + un1 = un32*b + un1 - * - * The statements inside the if-clause do not change the value - * of the left-hand side of (2), and the loop is only exited - * if q1*vn0 <= rhat*b + un1, so: - * - * (3) q1*vn1*b + q1*vn0 <= un32*b + un1 - * (4) q1*v <= un32*b + un1 - * (5) 0 <= un32*b + un1 - q1*v - * - * By (5) we are certain that the possible add-back step from - * Knuth's algorithm D is never required. - * - * Since the final quotient is less than 2**32, the following - * must be true: - * - * (6) un32*b + un1 - q1*v <= UINT32_MAX - * - * This means that in the following line, the high words - * of un32*b and q1*v can be discarded without any effect - * on the result. - */ - un21 = un32*b + un1 - q1*v; - - q0 = un21 / vn1; - rhat = un21 - q0*vn1; -again2: - if (q0 >= b || q0*vn0 > b*rhat + un0) { - q0 = q0 - 1; - rhat = rhat + vn1; - if (rhat < b) goto again2; - } - - *q = q1*b + q0; - *r = (un21*b + un0 - q0*v) >> s; -} -#endif /* END ANSI + LEGACY_COMPILER */ - -/* END ANSI */ -#elif defined(ASM) -static inline void -_mpd_mul_words(mpd_uint_t *hi, mpd_uint_t *lo, mpd_uint_t a, mpd_uint_t b) -{ - mpd_uint_t h, l; - - __asm__ ( "mull %3\n\t" - : "=d" (h), "=a" (l) - : "%a" (a), "rm" (b) - : "cc" - ); - - *hi = h; - *lo = l; -} - -static inline void -_mpd_div_words(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t hi, mpd_uint_t lo, - mpd_uint_t d) -{ - mpd_uint_t qq, rr; - - __asm__ ( "divl %4\n\t" - : "=a" (qq), "=d" (rr) - : "a" (lo), "d" (hi), "rm" (d) - : "cc" - ); - - *q = qq; - *r = rr; -} -/* END GCC ASM */ -#elif defined(MASM) -static inline void __cdecl -_mpd_mul_words(mpd_uint_t *hi, mpd_uint_t *lo, mpd_uint_t a, mpd_uint_t b) -{ - mpd_uint_t h, l; - - __asm { - mov eax, a - mul b - mov h, edx - mov l, eax - } - - *hi = h; - *lo = l; -} - -static inline void __cdecl -_mpd_div_words(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t hi, mpd_uint_t lo, - mpd_uint_t d) -{ - mpd_uint_t qq, rr; - - __asm { - mov eax, lo - mov edx, hi - div d - mov qq, eax - mov rr, edx - } - - *q = qq; - *r = rr; -} -/* END MASM (_MSC_VER) */ -#else - #error "need platform specific 64 bit multiplication and division" -#endif - -#define DIVMOD(q, r, v, d) *q = v / d; *r = v - *q * d -static inline void -_mpd_divmod_pow10(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t v, mpd_uint_t exp) -{ - assert(exp <= 9); - - if (exp <= 4) { - switch (exp) { - case 0: *q = v; *r = 0; break; - case 1: DIVMOD(q, r, v, 10UL); break; - case 2: DIVMOD(q, r, v, 100UL); break; - case 3: DIVMOD(q, r, v, 1000UL); break; - case 4: DIVMOD(q, r, v, 10000UL); break; - } - } - else { - switch (exp) { - case 5: DIVMOD(q, r, v, 100000UL); break; - case 6: DIVMOD(q, r, v, 1000000UL); break; - case 7: DIVMOD(q, r, v, 10000000UL); break; - case 8: DIVMOD(q, r, v, 100000000UL); break; - case 9: DIVMOD(q, r, v, 1000000000UL); break; /* GCOV_NOT_REACHED */ - } - } -} -/* END CONFIG_32 */ - -/* NO CONFIG */ -#else - #error "define CONFIG_64 or CONFIG_32" -#endif /* CONFIG */ - - -static inline void -_mpd_div_word(mpd_uint_t *q, mpd_uint_t *r, mpd_uint_t v, mpd_uint_t d) -{ - *q = v / d; - *r = v - *q * d; -} - -static inline void -_mpd_idiv_word(mpd_ssize_t *q, mpd_ssize_t *r, mpd_ssize_t v, mpd_ssize_t d) -{ - *q = v / d; - *r = v - *q * d; -} - - -/** ------------------------------------------------------------ - ** Arithmetic with overflow checking - ** ------------------------------------------------------------ - */ - -/* The following macros do call exit() in case of an overflow. - If the library is used correctly (i.e. with valid context - parameters), such overflows cannot occur. The macros are used - as sanity checks in a couple of strategic places and should - be viewed as a handwritten version of gcc's -ftrapv option. */ - -static inline mpd_size_t -add_size_t(mpd_size_t a, mpd_size_t b) -{ - if (a > MPD_SIZE_MAX - b) { - mpd_err_fatal("add_size_t(): overflow: check the context"); /* GCOV_NOT_REACHED */ - } - return a + b; -} - -static inline mpd_size_t -sub_size_t(mpd_size_t a, mpd_size_t b) -{ - if (b > a) { - mpd_err_fatal("sub_size_t(): overflow: check the context"); /* GCOV_NOT_REACHED */ - } - return a - b; -} - -#if MPD_SIZE_MAX != MPD_UINT_MAX - #error "adapt mul_size_t() and mulmod_size_t()" -#endif - -static inline mpd_size_t -mul_size_t(mpd_size_t a, mpd_size_t b) -{ - mpd_uint_t hi, lo; - - _mpd_mul_words(&hi, &lo, (mpd_uint_t)a, (mpd_uint_t)b); - if (hi) { - mpd_err_fatal("mul_size_t(): overflow: check the context"); /* GCOV_NOT_REACHED */ - } - return lo; -} - -static inline mpd_size_t -add_size_t_overflow(mpd_size_t a, mpd_size_t b, mpd_size_t *overflow) -{ - mpd_size_t ret; - - *overflow = 0; - ret = a + b; - if (ret < a) *overflow = 1; - return ret; -} - -static inline mpd_size_t -mul_size_t_overflow(mpd_size_t a, mpd_size_t b, mpd_size_t *overflow) -{ - mpd_uint_t hi, lo; - - _mpd_mul_words(&hi, &lo, (mpd_uint_t)a, (mpd_uint_t)b); - *overflow = (mpd_size_t)hi; - return lo; -} - -static inline mpd_ssize_t -mod_mpd_ssize_t(mpd_ssize_t a, mpd_ssize_t m) -{ - mpd_ssize_t r = a % m; - return (r < 0) ? r + m : r; -} - -static inline mpd_size_t -mulmod_size_t(mpd_size_t a, mpd_size_t b, mpd_size_t m) -{ - mpd_uint_t hi, lo; - mpd_uint_t q, r; - - _mpd_mul_words(&hi, &lo, (mpd_uint_t)a, (mpd_uint_t)b); - _mpd_div_words(&q, &r, hi, lo, (mpd_uint_t)m); - - return r; -} - - -#endif /* LIBMPDEC_TYPEARITH_H_ */ diff --git a/Modules/_decimal/libmpdec/umodarith.h b/Modules/_decimal/libmpdec/umodarith.h deleted file mode 100644 index d7dbbbe6a7331a8..000000000000000 --- a/Modules/_decimal/libmpdec/umodarith.h +++ /dev/null @@ -1,648 +0,0 @@ -/* - * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - - -#ifndef LIBMPDEC_UMODARITH_H_ -#define LIBMPDEC_UMODARITH_H_ - - -#include "mpdecimal.h" - -#include "constants.h" -#include "typearith.h" - - -/* Bignum: Low level routines for unsigned modular arithmetic. These are - used in the fast convolution functions for very large coefficients. */ - - -/**************************************************************************/ -/* ANSI modular arithmetic */ -/**************************************************************************/ - - -/* - * Restrictions: a < m and b < m - * ACL2 proof: umodarith.lisp: addmod-correct - */ -static inline mpd_uint_t -addmod(mpd_uint_t a, mpd_uint_t b, mpd_uint_t m) -{ - mpd_uint_t s; - - s = a + b; - s = (s < a) ? s - m : s; - s = (s >= m) ? s - m : s; - - return s; -} - -/* - * Restrictions: a < m and b < m - * ACL2 proof: umodarith.lisp: submod-2-correct - */ -static inline mpd_uint_t -submod(mpd_uint_t a, mpd_uint_t b, mpd_uint_t m) -{ - mpd_uint_t d; - - d = a - b; - d = (a < b) ? d + m : d; - - return d; -} - -/* - * Restrictions: a < 2m and b < 2m - * ACL2 proof: umodarith.lisp: section ext-submod - */ -static inline mpd_uint_t -ext_submod(mpd_uint_t a, mpd_uint_t b, mpd_uint_t m) -{ - mpd_uint_t d; - - a = (a >= m) ? a - m : a; - b = (b >= m) ? b - m : b; - - d = a - b; - d = (a < b) ? d + m : d; - - return d; -} - -/* - * Reduce double word modulo m. - * Restrictions: m != 0 - * ACL2 proof: umodarith.lisp: section dw-reduce - */ -static inline mpd_uint_t -dw_reduce(mpd_uint_t hi, mpd_uint_t lo, mpd_uint_t m) -{ - mpd_uint_t r1, r2, w; - - _mpd_div_word(&w, &r1, hi, m); - _mpd_div_words(&w, &r2, r1, lo, m); - - return r2; -} - -/* - * Subtract double word from a. - * Restrictions: a < m - * ACL2 proof: umodarith.lisp: section dw-submod - */ -static inline mpd_uint_t -dw_submod(mpd_uint_t a, mpd_uint_t hi, mpd_uint_t lo, mpd_uint_t m) -{ - mpd_uint_t d, r; - - r = dw_reduce(hi, lo, m); - d = a - r; - d = (a < r) ? d + m : d; - - return d; -} - -#ifdef CONFIG_64 - -/**************************************************************************/ -/* 64-bit modular arithmetic */ -/**************************************************************************/ - -/* - * A proof of the algorithm is in literature/mulmod-64.txt. An ACL2 - * proof is in umodarith.lisp: section "Fast modular reduction". - * - * Algorithm: calculate (a * b) % p: - * - * a) hi, lo <- a * b # Calculate a * b. - * - * b) hi, lo <- R(hi, lo) # Reduce modulo p. - * - * c) Repeat step b) until 0 <= hi * 2**64 + lo < 2*p. - * - * d) If the result is less than p, return lo. Otherwise return lo - p. - */ - -static inline mpd_uint_t -x64_mulmod(mpd_uint_t a, mpd_uint_t b, mpd_uint_t m) -{ - mpd_uint_t hi, lo, x, y; - - - _mpd_mul_words(&hi, &lo, a, b); - - if (m & (1ULL<<32)) { /* P1 */ - - /* first reduction */ - x = y = hi; - hi >>= 32; - - x = lo - x; - if (x > lo) hi--; - - y <<= 32; - lo = y + x; - if (lo < y) hi++; - - /* second reduction */ - x = y = hi; - hi >>= 32; - - x = lo - x; - if (x > lo) hi--; - - y <<= 32; - lo = y + x; - if (lo < y) hi++; - - return (hi || lo >= m ? lo - m : lo); - } - else if (m & (1ULL<<34)) { /* P2 */ - - /* first reduction */ - x = y = hi; - hi >>= 30; - - x = lo - x; - if (x > lo) hi--; - - y <<= 34; - lo = y + x; - if (lo < y) hi++; - - /* second reduction */ - x = y = hi; - hi >>= 30; - - x = lo - x; - if (x > lo) hi--; - - y <<= 34; - lo = y + x; - if (lo < y) hi++; - - /* third reduction */ - x = y = hi; - hi >>= 30; - - x = lo - x; - if (x > lo) hi--; - - y <<= 34; - lo = y + x; - if (lo < y) hi++; - - return (hi || lo >= m ? lo - m : lo); - } - else { /* P3 */ - - /* first reduction */ - x = y = hi; - hi >>= 24; - - x = lo - x; - if (x > lo) hi--; - - y <<= 40; - lo = y + x; - if (lo < y) hi++; - - /* second reduction */ - x = y = hi; - hi >>= 24; - - x = lo - x; - if (x > lo) hi--; - - y <<= 40; - lo = y + x; - if (lo < y) hi++; - - /* third reduction */ - x = y = hi; - hi >>= 24; - - x = lo - x; - if (x > lo) hi--; - - y <<= 40; - lo = y + x; - if (lo < y) hi++; - - return (hi || lo >= m ? lo - m : lo); - } -} - -static inline void -x64_mulmod2c(mpd_uint_t *a, mpd_uint_t *b, mpd_uint_t w, mpd_uint_t m) -{ - *a = x64_mulmod(*a, w, m); - *b = x64_mulmod(*b, w, m); -} - -static inline void -x64_mulmod2(mpd_uint_t *a0, mpd_uint_t b0, mpd_uint_t *a1, mpd_uint_t b1, - mpd_uint_t m) -{ - *a0 = x64_mulmod(*a0, b0, m); - *a1 = x64_mulmod(*a1, b1, m); -} - -static inline mpd_uint_t -x64_powmod(mpd_uint_t base, mpd_uint_t exp, mpd_uint_t umod) -{ - mpd_uint_t r = 1; - - while (exp > 0) { - if (exp & 1) - r = x64_mulmod(r, base, umod); - base = x64_mulmod(base, base, umod); - exp >>= 1; - } - - return r; -} - -/* END CONFIG_64 */ -#else /* CONFIG_32 */ - - -/**************************************************************************/ -/* 32-bit modular arithmetic */ -/**************************************************************************/ - -#if defined(ANSI) -#if !defined(LEGACY_COMPILER) -/* HAVE_UINT64_T */ -static inline mpd_uint_t -std_mulmod(mpd_uint_t a, mpd_uint_t b, mpd_uint_t m) -{ - return ((mpd_uuint_t) a * b) % m; -} - -static inline void -std_mulmod2c(mpd_uint_t *a, mpd_uint_t *b, mpd_uint_t w, mpd_uint_t m) -{ - *a = ((mpd_uuint_t) *a * w) % m; - *b = ((mpd_uuint_t) *b * w) % m; -} - -static inline void -std_mulmod2(mpd_uint_t *a0, mpd_uint_t b0, mpd_uint_t *a1, mpd_uint_t b1, - mpd_uint_t m) -{ - *a0 = ((mpd_uuint_t) *a0 * b0) % m; - *a1 = ((mpd_uuint_t) *a1 * b1) % m; -} -/* END HAVE_UINT64_T */ -#else -/* LEGACY_COMPILER */ -static inline mpd_uint_t -std_mulmod(mpd_uint_t a, mpd_uint_t b, mpd_uint_t m) -{ - mpd_uint_t hi, lo, q, r; - _mpd_mul_words(&hi, &lo, a, b); - _mpd_div_words(&q, &r, hi, lo, m); - return r; -} - -static inline void -std_mulmod2c(mpd_uint_t *a, mpd_uint_t *b, mpd_uint_t w, mpd_uint_t m) -{ - *a = std_mulmod(*a, w, m); - *b = std_mulmod(*b, w, m); -} - -static inline void -std_mulmod2(mpd_uint_t *a0, mpd_uint_t b0, mpd_uint_t *a1, mpd_uint_t b1, - mpd_uint_t m) -{ - *a0 = std_mulmod(*a0, b0, m); - *a1 = std_mulmod(*a1, b1, m); -} -/* END LEGACY_COMPILER */ -#endif - -static inline mpd_uint_t -std_powmod(mpd_uint_t base, mpd_uint_t exp, mpd_uint_t umod) -{ - mpd_uint_t r = 1; - - while (exp > 0) { - if (exp & 1) - r = std_mulmod(r, base, umod); - base = std_mulmod(base, base, umod); - exp >>= 1; - } - - return r; -} -#endif /* ANSI CONFIG_32 */ - - -/**************************************************************************/ -/* Pentium Pro modular arithmetic */ -/**************************************************************************/ - -/* - * A proof of the algorithm is in literature/mulmod-ppro.txt. The FPU - * control word must be set to 64-bit precision and truncation mode - * prior to using these functions. - * - * Algorithm: calculate (a * b) % p: - * - * p := prime < 2**31 - * pinv := (long double)1.0 / p (precalculated) - * - * a) n = a * b # Calculate exact product. - * b) qest = n * pinv # Calculate estimate for q = n / p. - * c) q = (qest+2**63)-2**63 # Truncate qest to the exact quotient. - * d) r = n - q * p # Calculate remainder. - * - * Remarks: - * - * - p = dmod and pinv = dinvmod. - * - dinvmod points to an array of three uint32_t, which is interpreted - * as an 80 bit long double by fldt. - * - Intel compilers prior to version 11 do not seem to handle the - * __GNUC__ inline assembly correctly. - * - random tests are provided in tests/extended/ppro_mulmod.c - */ - -#if defined(PPRO) -#if defined(ASM) - -/* Return (a * b) % dmod */ -static inline mpd_uint_t -ppro_mulmod(mpd_uint_t a, mpd_uint_t b, double *dmod, uint32_t *dinvmod) -{ - mpd_uint_t retval; - - __asm__ ( - "fildl %2\n\t" - "fildl %1\n\t" - "fmulp %%st, %%st(1)\n\t" - "fldt (%4)\n\t" - "fmul %%st(1), %%st\n\t" - "flds %5\n\t" - "fadd %%st, %%st(1)\n\t" - "fsubrp %%st, %%st(1)\n\t" - "fldl (%3)\n\t" - "fmulp %%st, %%st(1)\n\t" - "fsubrp %%st, %%st(1)\n\t" - "fistpl %0\n\t" - : "=m" (retval) - : "m" (a), "m" (b), "r" (dmod), "r" (dinvmod), "m" (MPD_TWO63) - : "st", "memory" - ); - - return retval; -} - -/* - * Two modular multiplications in parallel: - * *a0 = (*a0 * w) % dmod - * *a1 = (*a1 * w) % dmod - */ -static inline void -ppro_mulmod2c(mpd_uint_t *a0, mpd_uint_t *a1, mpd_uint_t w, - double *dmod, uint32_t *dinvmod) -{ - __asm__ ( - "fildl %2\n\t" - "fildl (%1)\n\t" - "fmul %%st(1), %%st\n\t" - "fxch %%st(1)\n\t" - "fildl (%0)\n\t" - "fmulp %%st, %%st(1) \n\t" - "fldt (%4)\n\t" - "flds %5\n\t" - "fld %%st(2)\n\t" - "fmul %%st(2)\n\t" - "fadd %%st(1)\n\t" - "fsub %%st(1)\n\t" - "fmull (%3)\n\t" - "fsubrp %%st, %%st(3)\n\t" - "fxch %%st(2)\n\t" - "fistpl (%0)\n\t" - "fmul %%st(2)\n\t" - "fadd %%st(1)\n\t" - "fsubp %%st, %%st(1)\n\t" - "fmull (%3)\n\t" - "fsubrp %%st, %%st(1)\n\t" - "fistpl (%1)\n\t" - : : "r" (a0), "r" (a1), "m" (w), - "r" (dmod), "r" (dinvmod), - "m" (MPD_TWO63) - : "st", "memory" - ); -} - -/* - * Two modular multiplications in parallel: - * *a0 = (*a0 * b0) % dmod - * *a1 = (*a1 * b1) % dmod - */ -static inline void -ppro_mulmod2(mpd_uint_t *a0, mpd_uint_t b0, mpd_uint_t *a1, mpd_uint_t b1, - double *dmod, uint32_t *dinvmod) -{ - __asm__ ( - "fildl %3\n\t" - "fildl (%2)\n\t" - "fmulp %%st, %%st(1)\n\t" - "fildl %1\n\t" - "fildl (%0)\n\t" - "fmulp %%st, %%st(1)\n\t" - "fldt (%5)\n\t" - "fld %%st(2)\n\t" - "fmul %%st(1), %%st\n\t" - "fxch %%st(1)\n\t" - "fmul %%st(2), %%st\n\t" - "flds %6\n\t" - "fldl (%4)\n\t" - "fxch %%st(3)\n\t" - "fadd %%st(1), %%st\n\t" - "fxch %%st(2)\n\t" - "fadd %%st(1), %%st\n\t" - "fxch %%st(2)\n\t" - "fsub %%st(1), %%st\n\t" - "fxch %%st(2)\n\t" - "fsubp %%st, %%st(1)\n\t" - "fxch %%st(1)\n\t" - "fmul %%st(2), %%st\n\t" - "fxch %%st(1)\n\t" - "fmulp %%st, %%st(2)\n\t" - "fsubrp %%st, %%st(3)\n\t" - "fsubrp %%st, %%st(1)\n\t" - "fxch %%st(1)\n\t" - "fistpl (%2)\n\t" - "fistpl (%0)\n\t" - : : "r" (a0), "m" (b0), "r" (a1), "m" (b1), - "r" (dmod), "r" (dinvmod), - "m" (MPD_TWO63) - : "st", "memory" - ); -} -/* END PPRO GCC ASM */ -#elif defined(MASM) - -/* Return (a * b) % dmod */ -static inline mpd_uint_t __cdecl -ppro_mulmod(mpd_uint_t a, mpd_uint_t b, double *dmod, uint32_t *dinvmod) -{ - mpd_uint_t retval; - - __asm { - mov eax, dinvmod - mov edx, dmod - fild b - fild a - fmulp st(1), st - fld TBYTE PTR [eax] - fmul st, st(1) - fld MPD_TWO63 - fadd st(1), st - fsubp st(1), st - fld QWORD PTR [edx] - fmulp st(1), st - fsubp st(1), st - fistp retval - } - - return retval; -} - -/* - * Two modular multiplications in parallel: - * *a0 = (*a0 * w) % dmod - * *a1 = (*a1 * w) % dmod - */ -static inline mpd_uint_t __cdecl -ppro_mulmod2c(mpd_uint_t *a0, mpd_uint_t *a1, mpd_uint_t w, - double *dmod, uint32_t *dinvmod) -{ - __asm { - mov ecx, dmod - mov edx, a1 - mov ebx, dinvmod - mov eax, a0 - fild w - fild DWORD PTR [edx] - fmul st, st(1) - fxch st(1) - fild DWORD PTR [eax] - fmulp st(1), st - fld TBYTE PTR [ebx] - fld MPD_TWO63 - fld st(2) - fmul st, st(2) - fadd st, st(1) - fsub st, st(1) - fmul QWORD PTR [ecx] - fsubp st(3), st - fxch st(2) - fistp DWORD PTR [eax] - fmul st, st(2) - fadd st, st(1) - fsubrp st(1), st - fmul QWORD PTR [ecx] - fsubp st(1), st - fistp DWORD PTR [edx] - } -} - -/* - * Two modular multiplications in parallel: - * *a0 = (*a0 * b0) % dmod - * *a1 = (*a1 * b1) % dmod - */ -static inline void __cdecl -ppro_mulmod2(mpd_uint_t *a0, mpd_uint_t b0, mpd_uint_t *a1, mpd_uint_t b1, - double *dmod, uint32_t *dinvmod) -{ - __asm { - mov ecx, dmod - mov edx, a1 - mov ebx, dinvmod - mov eax, a0 - fild b1 - fild DWORD PTR [edx] - fmulp st(1), st - fild b0 - fild DWORD PTR [eax] - fmulp st(1), st - fld TBYTE PTR [ebx] - fld st(2) - fmul st, st(1) - fxch st(1) - fmul st, st(2) - fld DWORD PTR MPD_TWO63 - fld QWORD PTR [ecx] - fxch st(3) - fadd st, st(1) - fxch st(2) - fadd st, st(1) - fxch st(2) - fsub st, st(1) - fxch st(2) - fsubrp st(1), st - fxch st(1) - fmul st, st(2) - fxch st(1) - fmulp st(2), st - fsubp st(3), st - fsubp st(1), st - fxch st(1) - fistp DWORD PTR [edx] - fistp DWORD PTR [eax] - } -} -#endif /* PPRO MASM (_MSC_VER) */ - - -/* Return (base ** exp) % dmod */ -static inline mpd_uint_t -ppro_powmod(mpd_uint_t base, mpd_uint_t exp, double *dmod, uint32_t *dinvmod) -{ - mpd_uint_t r = 1; - - while (exp > 0) { - if (exp & 1) - r = ppro_mulmod(r, base, dmod, dinvmod); - base = ppro_mulmod(base, base, dmod, dinvmod); - exp >>= 1; - } - - return r; -} -#endif /* PPRO */ -#endif /* CONFIG_32 */ - - -#endif /* LIBMPDEC_UMODARITH_H_ */ diff --git a/Modules/_decimal/libmpdec/vcdiv64.asm b/Modules/_decimal/libmpdec/vcdiv64.asm deleted file mode 100644 index 597e9ba9352c8fc..000000000000000 --- a/Modules/_decimal/libmpdec/vcdiv64.asm +++ /dev/null @@ -1,46 +0,0 @@ -; -; Copyright (c) 2008-2020 Stefan Krah. All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions -; are met: -; -; 1. Redistributions of source code must retain the above copyright -; notice, this list of conditions and the following disclaimer. -; -; 2. Redistributions in binary form must reproduce the above copyright -; notice, this list of conditions and the following disclaimer in the -; documentation and/or other materials provided with the distribution. -; -; THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND -; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -; ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -; OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -; HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -; SUCH DAMAGE. -; - - -PUBLIC _mpd_div_words -_TEXT SEGMENT -q$ = 8 -r$ = 16 -hi$ = 24 -lo$ = 32 -d$ = 40 -_mpd_div_words PROC - mov r10, rdx - mov rdx, r8 - mov rax, r9 - div QWORD PTR d$[rsp] - mov QWORD PTR [r10], rdx - mov QWORD PTR [rcx], rax - ret 0 -_mpd_div_words ENDP -_TEXT ENDS -END diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index cbd1e026df27227..f827274eeffba83 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2297,6 +2297,10 @@ elementiter_next(PyObject *op) return NULL; } if (it->gettext) { + if (elem->tag != Py_None && !PyUnicode_Check(elem->tag)) { + Py_DECREF(elem); + continue; + } text = element_get_text(elem); goto gettext; } @@ -2570,6 +2574,7 @@ treebuilder_dealloc(PyObject *self) /* helpers for handling of arbitrary element-like objects */ /*[clinic input] +@permit_long_summary _elementtree._set_factories comment_factory: object @@ -2584,7 +2589,7 @@ For internal use only. static PyObject * _elementtree__set_factories_impl(PyObject *module, PyObject *comment_factory, PyObject *pi_factory) -/*[clinic end generated code: output=813b408adee26535 input=99d17627aea7fb3b]*/ +/*[clinic end generated code: output=813b408adee26535 input=0f415cb6b821f768]*/ { elementtreestate *st = get_elementtree_state(module); PyObject *old; @@ -3735,8 +3740,12 @@ _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target, PyErr_NoMemory(); return -1; } - /* expat < 2.1.0 has no XML_SetHashSalt() */ - if (EXPAT(st, SetHashSalt) != NULL) { + // Prefer 16-byte entropy, only expat >= 2.8.0. See gh-149018 + if (EXPAT(st, SetHashSalt16Bytes) != NULL) { + EXPAT(st, SetHashSalt16Bytes)(self->parser, + _Py_HashSecret.expat.hashsalt16); + } + else if (EXPAT(st, SetHashSalt) != NULL) { EXPAT(st, SetHashSalt)(self->parser, (unsigned long)_Py_HashSecret.expat.hashsalt); } diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 19bdf3d47c2fad5..b4595c55d519b93 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -860,7 +860,8 @@ static PyMethodDef partial_methods[] = { {"__reduce__", partial_reduce, METH_NOARGS}, {"__setstate__", partial_setstate, METH_O}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("partial is generic over the wrapped function's return type")}, {NULL, NULL} /* sentinel */ }; @@ -1060,19 +1061,18 @@ _functools_cmp_to_key_impl(PyObject *module, PyObject *mycmp) /*[clinic input] @permit_long_summary -@permit_long_docstring_body _functools.reduce function as func: object iterable as seq: object / - initial as result: object = NULL + initial as result: object(c_default="NULL") = functools._initial_missing Apply a function of two arguments cumulatively to the items of an iterable, from left to right. -This effectively reduces the iterable to a single value. If initial is present, -it is placed before the items of the iterable in the calculation, and serves as -a default when the iterable is empty. +This effectively reduces the iterable to a single value. If initial is +present, it is placed before the items of the iterable in the +calculation, and serves as a default when the iterable is empty. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates ((((1 + 2) + 3) + 4) + 5). @@ -1081,7 +1081,7 @@ calculates ((((1 + 2) + 3) + 4) + 5). static PyObject * _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, PyObject *result) -/*[clinic end generated code: output=30d898fe1267c79d input=4ccfb74548ce5170]*/ +/*[clinic end generated code: output=30d898fe1267c79d input=ff4d5c73100e72e8]*/ { PyObject *args, *it; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index faffe8d28c5b5e7..20d482021656a50 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -520,14 +520,14 @@ _gdbm.gdbm.firstkey Return the starting key for the traversal. -It's possible to loop over every key in the database using this method -and the nextkey() method. The traversal is ordered by GDBM's internal -hash values, and won't be sorted by the key values. +It's possible to loop over every key in the database using this +method and the nextkey() method. The traversal is ordered by GDBM's +internal hash values, and won't be sorted by the key values. [clinic start generated code]*/ static PyObject * _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=139275e9c8b60827 input=aad5a7c886c542f5]*/ +/*[clinic end generated code: output=139275e9c8b60827 input=ba40f0d81eae0f35]*/ { PyObject *v; datum key; @@ -556,8 +556,8 @@ _gdbm.gdbm.nextkey Returns the key that follows key in the traversal. -The following code prints every key in the database db, without having -to create a list in memory that contains them all: +The following code prints every key in the database db, without +having to create a list in memory that contains them all: k = db.firstkey() while k is not None: @@ -568,7 +568,7 @@ to create a list in memory that contains them all: static PyObject * _gdbm_gdbm_nextkey_impl(gdbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length) -/*[clinic end generated code: output=c81a69300ef41766 input=181f1130d5bfeb1e]*/ +/*[clinic end generated code: output=c81a69300ef41766 input=78293a913b02387e]*/ { PyObject *v; datum dbm_key, nextkey; @@ -599,14 +599,14 @@ Reorganize the database. If you have carried out a lot of deletions and would like to shrink the space used by the GDBM file, this routine will reorganize the -database. GDBM will not shorten the length of a database file except -by using this reorganization; otherwise, deleted file space will be -kept and reused as new (key,value) pairs are added. +database. GDBM will not shorten the length of a database file +except by using this reorganization; otherwise, deleted file space +will be kept and reused as new (key,value) pairs are added. [clinic start generated code]*/ static PyObject * _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=d77c69e8e3dd644a input=3e3ca0d2ea787861]*/ +/*[clinic end generated code: output=d77c69e8e3dd644a input=d7fcf03051c6f7cd]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c index f94e0e2254a9128..7ab722e737c2ef7 100644 --- a/Modules/_hacl/Lib_Memzero0.c +++ b/Modules/_hacl/Lib_Memzero0.c @@ -31,7 +31,7 @@ #include <string.h> #endif -#if defined(__FreeBSD__) || defined(__NetBSD__) +#if defined(__CYGWIN__) || defined(__FreeBSD__) || defined(__NetBSD__) #include <strings.h> #endif @@ -57,7 +57,7 @@ void Lib_Memzero0_memzero0(void *dst, uint64_t len) { SecureZeroMemory(dst, len_); #elif defined(__APPLE__) && defined(__MACH__) && defined(APPLE_HAS_MEMSET_S) memset_s(dst, len_, 0, len_); - #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__FreeBSD__) || defined(__OpenBSD__) + #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__CYGWIN__) || defined(__FreeBSD__) || defined(__OpenBSD__) explicit_bzero(dst, len_); #elif defined(__NetBSD__) explicit_memset(dst, 0, len_); diff --git a/Modules/_hacl/libintvector.h b/Modules/_hacl/libintvector.h index 6db5253eee4f24c..75ec2e0589830b1 100644 --- a/Modules/_hacl/libintvector.h +++ b/Modules/_hacl/libintvector.h @@ -802,6 +802,18 @@ vector128 Lib_IntVector_Intrinsics_vec128_xor(vector128 x0, vector128 x1) { #if defined(HACL_CAN_COMPILE_VEC128) #include <altivec.h> + +/* GCC's AltiVec extension hijacks 'bool' as '__vector __bool int'. + Restore C99/C11 scalar bool for HACL* code. */ +#if defined(__GNUC__) && !defined(__clang__) +#undef bool +#define bool _Bool +#undef true +#define true 1 +#undef false +#define false 0 +#endif + #include <string.h> // for memcpy #include <stdint.h> diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index 72ceb27b7f70e8e..c73bd4ee90f4c55 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=8ba599b2f6c9701b3dc961db895b0856a2210f76 +expected_hacl_star_rev=504c2987452f87fe44bce9b9f12e19d6e051761f hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index fa3eceb74d16943..f895c9037485c43 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2369,18 +2369,17 @@ _hashlib_HMAC_digest_impl(HMACobject *self) /*[clinic input] @permit_long_summary -@permit_long_docstring_body _hashlib.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. -This may be used to exchange the value safely in email or other non-binary -environments. +This may be used to exchange the value safely in email or other +non-binary environments. [clinic start generated code]*/ static PyObject * _hashlib_HMAC_hexdigest_impl(HMACobject *self) -/*[clinic end generated code: output=80d825be1eaae6a7 input=5e48db83ab1a4d19]*/ +/*[clinic end generated code: output=80d825be1eaae6a7 input=e37a84c36a43787c]*/ { unsigned char buf[EVP_MAX_MD_SIZE]; Py_ssize_t n = _hmac_digest(self, buf); @@ -2540,8 +2539,8 @@ _hashlib.get_fips_mode -> int Determine the OpenSSL FIPS mode of operation. For OpenSSL 3.0.0 and newer it returns the state of the default provider -in the default OSSL context. It's not quite the same as FIPS_mode() but good -enough for unittests. +in the default OSSL context. It's not quite the same as FIPS_mode() but +good enough for unittests. Effectively any non-zero return value indicates FIPS mode; values other than 1 may have additional significance. @@ -2549,7 +2548,7 @@ values other than 1 may have additional significance. static int _hashlib_get_fips_mode_impl(PyObject *module) -/*[clinic end generated code: output=87eece1bab4d3fa9 input=2db61538c41c6fef]*/ +/*[clinic end generated code: output=87eece1bab4d3fa9 input=a6cdb6901421d122]*/ { #ifdef Py_HAS_OPENSSL3_SUPPORT diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index c705376f4edbf09..014c838694975d5 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -247,6 +247,7 @@ _heapq_heapreplace_impl(PyObject *module, PyObject *heap, PyObject *item) } /*[clinic input] +@permit_long_summary @critical_section heap _heapq.heappushpop @@ -262,7 +263,7 @@ a separate call to heappop(). static PyObject * _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=67231dc98ed5774f input=db05c81b1dd92c44]*/ +/*[clinic end generated code: output=67231dc98ed5774f input=491178a1c7d417ba]*/ { PyObject *returnitem; int cmp; @@ -593,13 +594,13 @@ _heapq.heappushpop_max Maxheap variant of heappushpop. -The combined action runs more efficiently than heappush_max() followed by -a separate call to heappop_max(). +The combined action runs more efficiently than heappush_max() +followed by a separate call to heappop_max(). [clinic start generated code]*/ static PyObject * _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=ff0019f0941aca0d input=24d0defa6fd6df4a]*/ +/*[clinic end generated code: output=ff0019f0941aca0d input=52030929667a4c08]*/ { PyObject *returnitem; int cmp; diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 3c356cb40d2bca6..3614890757d69da 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -921,7 +921,8 @@ static _channelends * _channelends_new(void) { _channelends *ends = GLOBAL_MALLOC(_channelends); - if (ends== NULL) { + if (ends == NULL) { + PyErr_NoMemory(); return NULL; } ends->numsendopen = 0; @@ -1115,6 +1116,7 @@ _channel_new(PyThread_type_lock mutex, struct _channeldefaults defaults) assert(check_unbound(defaults.unboundop)); _channel_state *chan = GLOBAL_MALLOC(_channel_state); if (chan == NULL) { + PyErr_NoMemory(); return NULL; } chan->mutex = mutex; @@ -1313,6 +1315,7 @@ _channelref_new(int64_t cid, _channel_state *chan) { _channelref *ref = GLOBAL_MALLOC(_channelref); if (ref == NULL) { + PyErr_NoMemory(); return NULL; } ref->cid = cid; @@ -1698,6 +1701,7 @@ _channel_set_closing(_channelref *ref, PyThread_type_lock mutex) { } chan->closing = GLOBAL_MALLOC(struct _channel_closing); if (chan->closing == NULL) { + PyErr_NoMemory(); goto done; } chan->closing->ref = ref; @@ -2586,6 +2590,7 @@ static PyObject * _channelid_from_xid(_PyXIData_t *data) { struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); + PyObject *cidobj = NULL; // It might not be imported yet, so we can't use _get_current_module(). PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); @@ -2595,11 +2600,10 @@ _channelid_from_xid(_PyXIData_t *data) assert(mod != Py_None); module_state *state = get_module_state(mod); if (state == NULL) { - return NULL; + goto done; } // Note that we do not preserve the "resolve" flag. - PyObject *cidobj = NULL; int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, _global_channels(), 0, 0, (channelid **)&cidobj); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 777b68547498847..d203ddba7d9c3c9 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1101,6 +1101,7 @@ queue_create(_queues *queues, Py_ssize_t maxsize, } int64_t qid = _queues_add(queues, queue); if (qid < 0) { + queue->alive = 0; _queue_clear(queue); GLOBAL_FREE(queue); } @@ -1363,6 +1364,7 @@ _queueobj_from_xid(_PyXIData_t *data) if (mod == NULL) { mod = PyImport_ImportModule(MODULE_NAME_STR); if (mod == NULL) { + Py_DECREF(qidobj); return NULL; } } @@ -1552,12 +1554,13 @@ _interpqueues.destroy Clear and destroy the queue. -Afterward attempts to use the queue will behave as though it never existed. +Afterward attempts to use the queue will behave as though it never +existed. [clinic start generated code]*/ static PyObject * _interpqueues_destroy_impl(PyObject *module, int64_t qid) -/*[clinic end generated code: output=46b35623f080cbff input=8632bba87f81e3e9]*/ +/*[clinic end generated code: output=46b35623f080cbff input=75136ad807e28677]*/ { int err = queue_destroy(&_globals.queues, qid); if (handle_queue_error(err, module, qid)) { diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 4c9be1d525d5871..15bfd35a80806ce 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -144,7 +144,7 @@ xibufferview_from_buffer(PyTypeObject *cls, Py_buffer *view, int64_t interpid) Py_buffer *copied = PyMem_RawMalloc(sizeof(Py_buffer)); if (copied == NULL) { - return NULL; + return PyErr_NoMemory(); } /* This steals the view->obj reference */ *copied = *view; @@ -152,7 +152,7 @@ xibufferview_from_buffer(PyTypeObject *cls, Py_buffer *view, int64_t interpid) xibufferview *self = PyObject_Malloc(sizeof(xibufferview)); if (self == NULL) { PyMem_RawFree(copied); - return NULL; + return PyErr_NoMemory(); } PyObject_Init(&self->base, cls); *self = (xibufferview){ @@ -277,6 +277,7 @@ _pybuffer_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { struct xibuffer *view = PyMem_RawMalloc(sizeof(struct xibuffer)); if (view == NULL) { + PyErr_NoMemory(); return -1; } view->used = 0; @@ -1219,7 +1220,8 @@ _interpreters.run_func Execute the body of the provided function in the identified interpreter. Code objects are also supported. In both cases, closures and args -are not supported. Methods and other callables are not supported either. +are not supported. Methods and other callables are not supported +either. (See _interpreters.exec().) [clinic start generated code]*/ @@ -1227,7 +1229,7 @@ are not supported. Methods and other callables are not supported either. static PyObject * _interpreters_run_func_impl(PyObject *module, PyObject *id, PyObject *func, PyObject *shared, int restricted) -/*[clinic end generated code: output=131f7202ca4a0c5e input=2d62bb9b9eaf4948]*/ +/*[clinic end generated code: output=131f7202ca4a0c5e input=162b29823b33d5cc]*/ { #define FUNCNAME MODULE_NAME_STR ".run_func" PyThreadState *tstate = _PyThreadState_GET(); @@ -1374,6 +1376,7 @@ _interpreters_is_running_impl(PyObject *module, PyObject *id, int restricted) /*[clinic input] +@permit_long_summary _interpreters.get_config id: object * @@ -1384,7 +1387,7 @@ Return a representation of the config used to initialize the interpreter. static PyObject * _interpreters_get_config_impl(PyObject *module, PyObject *id, int restricted) -/*[clinic end generated code: output=56773353b9b7224a input=59519a01c22d96d1]*/ +/*[clinic end generated code: output=56773353b9b7224a input=8272d9ea9e4fb42a]*/ { if (id == Py_None) { id = NULL; @@ -1490,19 +1493,19 @@ _interpreters_decref_impl(PyObject *module, PyObject *id, int restricted) /*[clinic input] -@permit_long_docstring_body _interpreters.capture_exception exc as exc_arg: object = None Return a snapshot of an exception. -If "exc" is None then the current exception, if any, is used (but not cleared). -The returned snapshot is the same as what _interpreters.exec() returns. +If "exc" is None then the current exception, if any, is used (but not +cleared). The returned snapshot is the same as what +_interpreters.exec() returns. [clinic start generated code]*/ static PyObject * _interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg) -/*[clinic end generated code: output=ef3f5393ef9c88a6 input=6c4dcb78fb722217]*/ +/*[clinic end generated code: output=ef3f5393ef9c88a6 input=4e6289f8f2a47b5b]*/ { PyObject *exc = exc_arg; if (exc == NULL || exc == Py_None) { @@ -1539,7 +1542,9 @@ _interpreters_capture_exception_impl(PyObject *module, PyObject *exc_arg) } finally: - _PyXI_FreeExcInfo(info); + if (info != NULL) { + _PyXI_FreeExcInfo(info); + } if (exc != exc_arg) { if (PyErr_Occurred()) { PyErr_SetRaisedException(exc); diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 32c55f8e225ed91..03e6fbe08889d47 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -70,7 +70,6 @@ PyDoc_STRVAR(module_doc, /*[clinic input] module _io -@permit_long_docstring_body _io.open file: object mode: str = "r" @@ -86,112 +85,113 @@ Open file and return a stream. Raise OSError upon failure. file is either a text or byte string giving the name (and the path if the file isn't in the current working directory) of the file to be opened or an integer file descriptor of the file to be -wrapped. (If a file descriptor is given, it is closed when the +wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) mode is an optional string that specifies the mode in which the file -is opened. It defaults to 'r' which means open for reading in text +is opened. It defaults to 'r' which means open for reading in text mode. Other common values are 'w' for writing (truncating the file if it already exists), 'x' for creating and writing to a new file, and 'a' for appending (which on some Unix systems, means that all writes append to the end of the file regardless of the current seek position). In text mode, if encoding is not specified the encoding used is platform -dependent: locale.getencoding() is called to get the current locale encoding. -(For reading and writing raw bytes use binary mode and leave encoding -unspecified.) The available modes are: +dependent: locale.getencoding() is called to get the current locale +encoding. (For reading and writing raw bytes use binary mode and leave +encoding unspecified.) The available modes are: -========= =============================================================== +========= ========================================================== Character Meaning ---------- --------------------------------------------------------------- +--------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing -'a' open for writing, appending to the end of the file if it exists +'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) -========= =============================================================== +========= ========================================================== -The default mode is 'rt' (open for reading text). For binary random +The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while -'r+b' opens the file without truncation. The 'x' mode implies 'w' and +'r+b' opens the file without truncation. The 'x' mode implies 'w' and raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, -even when the underlying operating system doesn't. Files opened in +even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as -bytes objects without any decoding. In text mode (the default, or when +bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. -Pass 0 to switch buffering off (only allowed in binary mode), 1 to select -line buffering (only usable in text mode), and an integer > 1 to indicate -the size of a fixed-size chunk buffer. When no buffering argument is -given, the default buffering policy works as follows: +Pass 0 to switch buffering off (only allowed in binary mode), 1 to +select line buffering (only usable in text mode), and an integer > 1 to +indicate the size of a fixed-size chunk buffer. When no buffering +argument is given, the default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer - is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) - when the device block size is available. - On most systems, the buffer will typically be 128 kilobytes long. + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above for binary files. encoding is the name of the encoding used to decode or encode the -file. This should only be used in text mode. The default encoding is +file. This should only be used in text mode. The default encoding is platform dependent, but any encoding supported by Python can be passed. See the codecs module for the list of supported encodings. errors is an optional string that specifies how encoding errors are to -be handled---this argument should not be used in binary mode. Pass +be handled---this argument should not be used in binary mode. Pass 'strict' to raise a ValueError exception if there is an encoding error (the default of None has the same effect), or pass 'ignore' to ignore -errors. (Note that ignoring encoding errors can lead to data loss.) +errors. (Note that ignoring encoding errors can lead to data loss.) See the documentation for codecs.register or run 'help(codecs.Codec)' for a list of the permitted encoding error strings. newline controls how universal newlines works (it only applies to text -mode). It can be None, '', '\n', '\r', and '\r\n'. It works as +mode). It can be None, '', '\n', '\r', and '\r\n'. It works as follows: -* On input, if newline is None, universal newlines mode is - enabled. Lines in the input can end in '\n', '\r', or '\r\n', and - these are translated into '\n' before being returned to the - caller. If it is '', universal newline mode is enabled, but line - endings are returned to the caller untranslated. If it has any of - the other legal values, input lines are only terminated by the given - string, and the line ending is returned to the caller untranslated. +* On input, if newline is None, universal newlines mode is enabled. + Lines in the input can end in '\n', '\r', or '\r\n', and these are + translated into '\n' before being returned to the caller. If it is + '', universal newline mode is enabled, but line endings are returned + to the caller untranslated. If it has any of the other legal values, + input lines are only terminated by the given string, and the line + ending is returned to the caller untranslated. * On output, if newline is None, any '\n' characters written are - translated to the system default line separator, os.linesep. If - newline is '' or '\n', no translation takes place. If newline is any + translated to the system default line separator, os.linesep. If + newline is '' or '\n', no translation takes place. If newline is any of the other legal values, any '\n' characters written are translated to the given string. If closefd is False, the underlying file descriptor will be kept open -when the file is closed. This does not work when a file name is given +when the file is closed. This does not work when a file name is given and must be True in that case. -A custom opener can be used by passing a callable as *opener*. The +A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by -calling *opener* with (*file*, *flags*). *opener* must return an open +calling *opener* with (*file*, *flags*). *opener* must return an open file descriptor (passing os.open as *opener* results in functionality similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing -are performed. When open() is used to open a file in a text mode ('w', -'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open +are performed. When open() is used to open a file in a text mode ('w', +'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open a file in a binary mode, the returned class varies: in read binary mode, it returns a BufferedReader; in write binary and append binary modes, it returns a BufferedWriter, and in read/write mode, it returns a BufferedRandom. It is also possible to use a string or bytearray as a file for both -reading and writing. For strings StringIO can be used like a file +reading and writing. For strings StringIO can be used like a file opened in a text mode, and for bytes a BytesIO can be used like a file opened in a binary mode. [clinic start generated code]*/ @@ -200,7 +200,7 @@ static PyObject * _io_open_impl(PyObject *module, PyObject *file, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd, PyObject *opener) -/*[clinic end generated code: output=aefafc4ce2b46dc0 input=8629579a442a99e3]*/ +/*[clinic end generated code: output=aefafc4ce2b46dc0 input=b3cefa70bef404b3]*/ { size_t i; @@ -499,21 +499,20 @@ _io_text_encoding_impl(PyObject *module, PyObject *encoding, int stacklevel) /*[clinic input] -@permit_long_docstring_body _io.open_code path : unicode Opens the provided file with the intent to import the contents. -This may perform extra validation beyond open(), but is otherwise interchangeable -with calling open(path, 'rb'). +This may perform extra validation beyond open(), but is otherwise +interchangeable with calling open(path, 'rb'). [clinic start generated code]*/ static PyObject * _io_open_code_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=53d38a37d780d034]*/ +/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=2803c35aeb63c719]*/ { return PyFile_OpenCodeObject(path); } diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 0fdae7b2d210040..5537947f6a51c11 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1509,7 +1509,9 @@ buffered_iternext(PyObject *op) tp == state->PyBufferedRandom_Type) { /* Skip method call overhead for speed */ + Py_BEGIN_CRITICAL_SECTION(self); line = _buffered_readline(self, -1); + Py_END_CRITICAL_SECTION(); } else { line = PyObject_CallMethodNoArgs((PyObject *)self, diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index d088bb0efac797a..8cdcbd0d89c718e 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -488,13 +488,13 @@ _io.BytesIO.read1 Read at most size bytes, returned as a bytes object. -If the size argument is negative or omitted, read until EOF is reached. -Return an empty bytes object at EOF. +If the size argument is negative or omitted, read until EOF is +reached. Return an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=d0f843285aa95f1c input=a08fc9e507ab380c]*/ +/*[clinic end generated code: output=d0f843285aa95f1c input=796ff4e0efccc4d9]*/ { return _io_BytesIO_read_impl(self, size); } @@ -792,13 +792,13 @@ _io.BytesIO.writelines Write lines to the file. Note that newlines are not added. lines can be any iterable object -producing bytes-like objects. This is equivalent to calling write() for -each element. +producing bytes-like objects. This is equivalent to calling write() +for each element. [clinic start generated code]*/ static PyObject * _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) -/*[clinic end generated code: output=03a43a75773bc397 input=5d6a616ae39dc9ca]*/ +/*[clinic end generated code: output=03a43a75773bc397 input=d265f76533b058e7]*/ { PyObject *it, *item; diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 90b80af3018fb0f..f03638064385e28 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -18,112 +18,113 @@ PyDoc_STRVAR(_io_open__doc__, "file is either a text or byte string giving the name (and the path\n" "if the file isn\'t in the current working directory) of the file to\n" "be opened or an integer file descriptor of the file to be\n" -"wrapped. (If a file descriptor is given, it is closed when the\n" +"wrapped. (If a file descriptor is given, it is closed when the\n" "returned I/O object is closed, unless closefd is set to False.)\n" "\n" "mode is an optional string that specifies the mode in which the file\n" -"is opened. It defaults to \'r\' which means open for reading in text\n" +"is opened. It defaults to \'r\' which means open for reading in text\n" "mode. Other common values are \'w\' for writing (truncating the file if\n" "it already exists), \'x\' for creating and writing to a new file, and\n" "\'a\' for appending (which on some Unix systems, means that all writes\n" "append to the end of the file regardless of the current seek position).\n" "In text mode, if encoding is not specified the encoding used is platform\n" -"dependent: locale.getencoding() is called to get the current locale encoding.\n" -"(For reading and writing raw bytes use binary mode and leave encoding\n" -"unspecified.) The available modes are:\n" +"dependent: locale.getencoding() is called to get the current locale\n" +"encoding. (For reading and writing raw bytes use binary mode and leave\n" +"encoding unspecified.) The available modes are:\n" "\n" -"========= ===============================================================\n" +"========= ==========================================================\n" "Character Meaning\n" -"--------- ---------------------------------------------------------------\n" +"--------- ----------------------------------------------------------\n" "\'r\' open for reading (default)\n" "\'w\' open for writing, truncating the file first\n" "\'x\' create a new file and open it for writing\n" -"\'a\' open for writing, appending to the end of the file if it exists\n" +"\'a\' open for writing, appending to the end of the file if it\n" +" exists\n" "\'b\' binary mode\n" "\'t\' text mode (default)\n" "\'+\' open a disk file for updating (reading and writing)\n" -"========= ===============================================================\n" +"========= ==========================================================\n" "\n" -"The default mode is \'rt\' (open for reading text). For binary random\n" +"The default mode is \'rt\' (open for reading text). For binary random\n" "access, the mode \'w+b\' opens and truncates the file to 0 bytes, while\n" -"\'r+b\' opens the file without truncation. The \'x\' mode implies \'w\' and\n" +"\'r+b\' opens the file without truncation. The \'x\' mode implies \'w\' and\n" "raises an `FileExistsError` if the file already exists.\n" "\n" "Python distinguishes between files opened in binary and text modes,\n" -"even when the underlying operating system doesn\'t. Files opened in\n" +"even when the underlying operating system doesn\'t. Files opened in\n" "binary mode (appending \'b\' to the mode argument) return contents as\n" -"bytes objects without any decoding. In text mode (the default, or when\n" +"bytes objects without any decoding. In text mode (the default, or when\n" "\'t\' is appended to the mode argument), the contents of the file are\n" "returned as strings, the bytes having been first decoded using a\n" "platform-dependent encoding or using the specified encoding if given.\n" "\n" "buffering is an optional integer used to set the buffering policy.\n" -"Pass 0 to switch buffering off (only allowed in binary mode), 1 to select\n" -"line buffering (only usable in text mode), and an integer > 1 to indicate\n" -"the size of a fixed-size chunk buffer. When no buffering argument is\n" -"given, the default buffering policy works as follows:\n" +"Pass 0 to switch buffering off (only allowed in binary mode), 1 to\n" +"select line buffering (only usable in text mode), and an integer > 1 to\n" +"indicate the size of a fixed-size chunk buffer. When no buffering\n" +"argument is given, the default buffering policy works as follows:\n" "\n" "* Binary files are buffered in fixed-size chunks; the size of the buffer\n" -" is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)\n" -" when the device block size is available.\n" -" On most systems, the buffer will typically be 128 kilobytes long.\n" +" is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device\n" +" block size is available.\n" +" On most systems, the buffer will typically be 128 kilobytes long.\n" "\n" "* \"Interactive\" text files (files for which isatty() returns True)\n" " use line buffering. Other text files use the policy described above\n" " for binary files.\n" "\n" "encoding is the name of the encoding used to decode or encode the\n" -"file. This should only be used in text mode. The default encoding is\n" +"file. This should only be used in text mode. The default encoding is\n" "platform dependent, but any encoding supported by Python can be\n" "passed. See the codecs module for the list of supported encodings.\n" "\n" "errors is an optional string that specifies how encoding errors are to\n" -"be handled---this argument should not be used in binary mode. Pass\n" +"be handled---this argument should not be used in binary mode. Pass\n" "\'strict\' to raise a ValueError exception if there is an encoding error\n" "(the default of None has the same effect), or pass \'ignore\' to ignore\n" -"errors. (Note that ignoring encoding errors can lead to data loss.)\n" +"errors. (Note that ignoring encoding errors can lead to data loss.)\n" "See the documentation for codecs.register or run \'help(codecs.Codec)\'\n" "for a list of the permitted encoding error strings.\n" "\n" "newline controls how universal newlines works (it only applies to text\n" -"mode). It can be None, \'\', \'\\n\', \'\\r\', and \'\\r\\n\'. It works as\n" +"mode). It can be None, \'\', \'\\n\', \'\\r\', and \'\\r\\n\'. It works as\n" "follows:\n" "\n" -"* On input, if newline is None, universal newlines mode is\n" -" enabled. Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\', and\n" -" these are translated into \'\\n\' before being returned to the\n" -" caller. If it is \'\', universal newline mode is enabled, but line\n" -" endings are returned to the caller untranslated. If it has any of\n" -" the other legal values, input lines are only terminated by the given\n" -" string, and the line ending is returned to the caller untranslated.\n" +"* On input, if newline is None, universal newlines mode is enabled.\n" +" Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\', and these are\n" +" translated into \'\\n\' before being returned to the caller. If it is\n" +" \'\', universal newline mode is enabled, but line endings are returned\n" +" to the caller untranslated. If it has any of the other legal values,\n" +" input lines are only terminated by the given string, and the line\n" +" ending is returned to the caller untranslated.\n" "\n" "* On output, if newline is None, any \'\\n\' characters written are\n" -" translated to the system default line separator, os.linesep. If\n" -" newline is \'\' or \'\\n\', no translation takes place. If newline is any\n" +" translated to the system default line separator, os.linesep. If\n" +" newline is \'\' or \'\\n\', no translation takes place. If newline is any\n" " of the other legal values, any \'\\n\' characters written are translated\n" " to the given string.\n" "\n" "If closefd is False, the underlying file descriptor will be kept open\n" -"when the file is closed. This does not work when a file name is given\n" +"when the file is closed. This does not work when a file name is given\n" "and must be True in that case.\n" "\n" -"A custom opener can be used by passing a callable as *opener*. The\n" +"A custom opener can be used by passing a callable as *opener*. The\n" "underlying file descriptor for the file object is then obtained by\n" -"calling *opener* with (*file*, *flags*). *opener* must return an open\n" +"calling *opener* with (*file*, *flags*). *opener* must return an open\n" "file descriptor (passing os.open as *opener* results in functionality\n" "similar to passing None).\n" "\n" "open() returns a file object whose type depends on the mode, and\n" "through which the standard file operations such as reading and writing\n" -"are performed. When open() is used to open a file in a text mode (\'w\',\n" -"\'r\', \'wt\', \'rt\', etc.), it returns a TextIOWrapper. When used to open\n" +"are performed. When open() is used to open a file in a text mode (\'w\',\n" +"\'r\', \'wt\', \'rt\', etc.), it returns a TextIOWrapper. When used to open\n" "a file in a binary mode, the returned class varies: in read binary\n" "mode, it returns a BufferedReader; in write binary and append binary\n" "modes, it returns a BufferedWriter, and in read/write mode, it returns\n" "a BufferedRandom.\n" "\n" "It is also possible to use a string or bytearray as a file for both\n" -"reading and writing. For strings StringIO can be used like a file\n" +"reading and writing. For strings StringIO can be used like a file\n" "opened in a text mode, and for bytes a BytesIO can be used like a file\n" "opened in a binary mode."); @@ -352,8 +353,8 @@ PyDoc_STRVAR(_io_open_code__doc__, "\n" "Opens the provided file with the intent to import the contents.\n" "\n" -"This may perform extra validation beyond open(), but is otherwise interchangeable\n" -"with calling open(path, \'rb\')."); +"This may perform extra validation beyond open(), but is otherwise\n" +"interchangeable with calling open(path, \'rb\')."); #define _IO_OPEN_CODE_METHODDEF \ {"open_code", _PyCFunction_CAST(_io_open_code), METH_FASTCALL|METH_KEYWORDS, _io_open_code__doc__}, @@ -410,4 +411,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=7a8e032c0424bce2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5190d11f0803bfe8 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index 6595dc937bbcf0f..fad11ea6c9f6cf6 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -254,8 +254,8 @@ PyDoc_STRVAR(_io_BytesIO_read1__doc__, "\n" "Read at most size bytes, returned as a bytes object.\n" "\n" -"If the size argument is negative or omitted, read until EOF is reached.\n" -"Return an empty bytes object at EOF."); +"If the size argument is negative or omitted, read until EOF is\n" +"reached. Return an empty bytes object at EOF."); #define _IO_BYTESIO_READ1_METHODDEF \ {"read1", _PyCFunction_CAST(_io_BytesIO_read1), METH_FASTCALL, _io_BytesIO_read1__doc__}, @@ -529,8 +529,8 @@ PyDoc_STRVAR(_io_BytesIO_writelines__doc__, "Write lines to the file.\n" "\n" "Note that newlines are not added. lines can be any iterable object\n" -"producing bytes-like objects. This is equivalent to calling write() for\n" -"each element."); +"producing bytes-like objects. This is equivalent to calling write()\n" +"for each element."); #define _IO_BYTESIO_WRITELINES_METHODDEF \ {"writelines", (PyCFunction)_io_BytesIO_writelines, METH_O, _io_BytesIO_writelines__doc__}, @@ -637,4 +637,4 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=daa81dfdae5ccc57 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eac3911e207aaf45 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 96c31ce8d6f415a..890b6bc3fac9d55 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -15,8 +15,8 @@ PyDoc_STRVAR(_io_FileIO_close__doc__, "\n" "Close the file.\n" "\n" -"A closed file cannot be used for further I/O operations. close() may be\n" -"called more than once without error."); +"A closed file cannot be used for further I/O operations. close()\n" +"may be called more than once without error."); #define _IO_FILEIO_CLOSE_METHODDEF \ {"close", _PyCFunction_CAST(_io_FileIO_close), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_close__doc__}, @@ -41,16 +41,19 @@ PyDoc_STRVAR(_io_FileIO___init____doc__, "Open a file.\n" "\n" "The mode can be \'r\' (default), \'w\', \'x\' or \'a\' for reading,\n" -"writing, exclusive creation or appending. The file will be created if it\n" -"doesn\'t exist when opened for writing or appending; it will be truncated\n" -"when opened for writing. A FileExistsError will be raised if it already\n" -"exists when opened for creating. Opening a file for creating implies\n" -"writing so this mode behaves in a similar way to \'w\'.Add a \'+\' to the mode\n" -"to allow simultaneous reading and writing. A custom opener can be used by\n" -"passing a callable as *opener*. The underlying file descriptor for the file\n" -"object is then obtained by calling opener with (*name*, *flags*).\n" -"*opener* must return an open file descriptor (passing os.open as *opener*\n" -"results in functionality similar to passing None)."); +"writing, exclusive creation or appending. The file will be created\n" +"if it doesn\'t exist when opened for writing or appending; it will be\n" +"truncated when opened for writing. A FileExistsError will be raised\n" +"if it already exists when opened for creating. Opening a file for\n" +"creating implies writing so this mode behaves in a similar way to\n" +"\'w\'. Add a \'+\' to the mode to allow simultaneous reading and\n" +"writing.\n" +"\n" +"A custom opener can be used by passing a callable as *opener*.\n" +"The underlying file descriptor for the file object is then obtained\n" +"by calling opener with (*name*, *flags*). *opener* must return\n" +"an open file descriptor (passing os.open as *opener* results in\n" +"functionality similar to passing None)."); static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, @@ -270,11 +273,13 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "\n" "Read all data from the file, returned as bytes.\n" "\n" -"Reads until either there is an error or read() returns size 0 (indicates EOF).\n" -"If the file is already at EOF, returns an empty bytes object.\n" +"Reads until either there is an error or read() returns size 0\n" +"(indicates EOF). If the file is already at EOF, returns an empty\n" +"bytes object.\n" "\n" -"In non-blocking mode, returns as much data as could be read before EAGAIN. If no\n" -"data is available (EAGAIN is returned before bytes are read) returns None."); +"In non-blocking mode, returns as much data as could be read before\n" +"EAGAIN. If no data is available (EAGAIN is returned before bytes\n" +"are read) returns None."); #define _IO_FILEIO_READALL_METHODDEF \ {"readall", _PyCFunction_CAST(_io_FileIO_readall), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_readall__doc__}, @@ -298,14 +303,14 @@ PyDoc_STRVAR(_io_FileIO_read__doc__, "\n" "Read at most size bytes, returned as bytes.\n" "\n" -"If size is less than 0, read all bytes in the file making multiple read calls.\n" -"See ``FileIO.readall``.\n" +"If size is less than 0, read all bytes in the file making multiple\n" +"read calls. See ``FileIO.readall``.\n" "\n" -"Attempts to make only one system call, retrying only per PEP 475 (EINTR). This\n" -"means less data may be returned than requested.\n" +"Attempts to make only one system call, retrying only per PEP 475\n" +"(EINTR). This means less data may be returned than requested.\n" "\n" -"In non-blocking mode, returns None if no data is available. Return an empty\n" -"bytes object at EOF."); +"In non-blocking mode, returns None if no data is available. Return\n" +"an empty bytes object at EOF."); #define _IO_FILEIO_READ_METHODDEF \ {"read", _PyCFunction_CAST(_io_FileIO_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_read__doc__}, @@ -358,8 +363,8 @@ PyDoc_STRVAR(_io_FileIO_write__doc__, "Write buffer b to file, return number of bytes written.\n" "\n" "Only makes one system call, so not all of the data may be written.\n" -"The number of bytes actually written is returned. In non-blocking mode,\n" -"returns None if the write would block."); +"The number of bytes actually written is returned. In non-blocking\n" +"mode, returns None if the write would block."); #define _IO_FILEIO_WRITE_METHODDEF \ {"write", _PyCFunction_CAST(_io_FileIO_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_write__doc__}, @@ -412,11 +417,12 @@ PyDoc_STRVAR(_io_FileIO_seek__doc__, "\n" "Move to new file position and return the file position.\n" "\n" -"Argument offset is a byte count. Optional argument whence defaults to\n" -"SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values\n" -"are SEEK_CUR or 1 (move relative to current position, positive or negative),\n" -"and SEEK_END or 2 (move relative to end of file, usually negative, although\n" -"many platforms allow seeking beyond the end of a file).\n" +"Argument offset is a byte count. Optional argument whence defaults\n" +"to SEEK_SET or 0 (offset from start of file, offset should be >= 0);\n" +"other values are SEEK_CUR or 1 (move relative to current position,\n" +"positive or negative), and SEEK_END or 2 (move relative to end of\n" +"file, usually negative, although many platforms allow seeking beyond\n" +"the end of a file).\n" "\n" "Note that not all file objects are seekable."); @@ -547,4 +553,4 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=2e48f3df2f189170 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=453d584e2e72f986 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 402448545dfc516..e4438c26431aa8e 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -19,11 +19,13 @@ PyDoc_STRVAR(_io__IOBase_seek__doc__, " whence\n" " The relative position to seek from.\n" "\n" -"The offset is interpreted relative to the position indicated by whence.\n" -"Values for whence are:\n" +"The offset is interpreted relative to the position indicated by\n" +"whence. Values for whence are:\n" "\n" -"* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n" -"* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n" +"* os.SEEK_SET or 0 -- start of stream (the default); offset should\n" +" be zero or positive\n" +"* os.SEEK_CUR or 1 -- current stream position; offset may be\n" +" negative\n" "* os.SEEK_END or 2 -- end of stream; offset is usually negative\n" "\n" "Return the new absolute position."); @@ -103,8 +105,8 @@ PyDoc_STRVAR(_io__IOBase_truncate__doc__, "\n" "Truncate file to size bytes.\n" "\n" -"File pointer is left unchanged. Size defaults to the current IO position\n" -"as reported by tell(). Return the new size."); +"File pointer is left unchanged. Size defaults to the current IO\n" +"position as reported by tell(). Return the new size."); #define _IO__IOBASE_TRUNCATE_METHODDEF \ {"truncate", _PyCFunction_CAST(_io__IOBase_truncate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_truncate__doc__}, @@ -443,4 +445,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=9359e74d95534bef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=28c06bb6db32c096 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index 83165e5f7ad08bf..d6d4afb9b63c624 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -179,7 +179,8 @@ PyDoc_STRVAR(_io_StringIO_seek__doc__, "\n" "Change stream position.\n" "\n" -"Seek to character offset pos relative to position indicated by whence:\n" +"Seek to character offset pos relative to position indicated by\n" +"whence:\n" " 0 Start of stream (the default). pos should be >= 0;\n" " 1 Current position - pos must be 0;\n" " 2 End of stream - pos must be 0.\n" @@ -550,4 +551,4 @@ _io_StringIO_newlines_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=bccc25ef8e6ce9ef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=730c34b2a6c0500b input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 3898a9c29824364..8d59bda5f74b386 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -16,7 +16,8 @@ PyDoc_STRVAR(_io__TextIOBase_detach__doc__, "\n" "Separate the underlying buffer from the TextIOBase and return it.\n" "\n" -"After the underlying buffer has been detached, the TextIO is in an unusable state."); +"After the underlying buffer has been detached, the TextIO is in\n" +"an unusable state."); #define _IO__TEXTIOBASE_DETACH_METHODDEF \ {"detach", _PyCFunction_CAST(_io__TextIOBase_detach), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_detach__doc__}, @@ -40,8 +41,8 @@ PyDoc_STRVAR(_io__TextIOBase_read__doc__, "\n" "Read at most size characters from stream.\n" "\n" -"Read from underlying buffer until we have size characters or we hit EOF.\n" -"If size is negative or omitted, read until EOF."); +"Read from underlying buffer until we have size characters or we hit\n" +"EOF. If size is negative or omitted, read until EOF."); #define _IO__TEXTIOBASE_READ_METHODDEF \ {"read", _PyCFunction_CAST(_io__TextIOBase_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_read__doc__}, @@ -665,7 +666,9 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper___init___impl((textio *)self, buffer, encoding, errors, newline, line_buffering, write_through); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -964,8 +967,8 @@ PyDoc_STRVAR(_io_TextIOWrapper_tell__doc__, "\n" "Return the stream position as an opaque number.\n" "\n" -"The return value of tell() can be given as input to seek(), to restore a\n" -"previous stream position."); +"The return value of tell() can be given as input to seek(), to\n" +"restore a previous stream position."); #define _IO_TEXTIOWRAPPER_TELL_METHODDEF \ {"tell", (PyCFunction)_io_TextIOWrapper_tell, METH_NOARGS, _io_TextIOWrapper_tell__doc__}, @@ -1328,4 +1331,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(PyObject *self, PyObject *value, void *Py_UNUS return return_value; } -/*[clinic end generated code: output=c38e6cd5ff4b7eea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8c571c9dba87d2b1 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 7af5923b6c17473..bd8073cd0af3f64 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -46,9 +46,9 @@ PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, "\n" "Open a console buffer by file descriptor.\n" "\n" -"The mode can be \'rb\' (default), or \'wb\' for reading or writing bytes. All\n" -"other mode characters will be ignored. Mode \'b\' will be assumed if it is\n" -"omitted. The *opener* parameter is always ignored."); +"The mode can be \'rb\' (default), or \'wb\' for reading or writing\n" +"bytes. All other mode characters will be ignored. Mode \'b\' will be\n" +"assumed if it is omitted. The *opener* parameter is always ignored."); static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, @@ -463,4 +463,4 @@ _io__WindowsConsoleIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=ce50bcd905f1f213 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dfe49dd71f4f4b1d input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 5d7741fdd830a53..3aeb30dfe24a357 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -153,13 +153,13 @@ _io.FileIO.close Close the file. -A closed file cannot be used for further I/O operations. close() may be -called more than once without error. +A closed file cannot be used for further I/O operations. close() +may be called more than once without error. [clinic start generated code]*/ static PyObject * _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) -/*[clinic end generated code: output=c30cbe9d1f23ca58 input=70da49e63db7c64d]*/ +/*[clinic end generated code: output=c30cbe9d1f23ca58 input=b405751dc4163da3]*/ { PyObject *res; int rc; @@ -231,22 +231,25 @@ _io.FileIO.__init__ Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, -writing, exclusive creation or appending. The file will be created if it -doesn't exist when opened for writing or appending; it will be truncated -when opened for writing. A FileExistsError will be raised if it already -exists when opened for creating. Opening a file for creating implies -writing so this mode behaves in a similar way to 'w'.Add a '+' to the mode -to allow simultaneous reading and writing. A custom opener can be used by -passing a callable as *opener*. The underlying file descriptor for the file -object is then obtained by calling opener with (*name*, *flags*). -*opener* must return an open file descriptor (passing os.open as *opener* -results in functionality similar to passing None). +writing, exclusive creation or appending. The file will be created +if it doesn't exist when opened for writing or appending; it will be +truncated when opened for writing. A FileExistsError will be raised +if it already exists when opened for creating. Opening a file for +creating implies writing so this mode behaves in a similar way to +'w'. Add a '+' to the mode to allow simultaneous reading and +writing. + +A custom opener can be used by passing a callable as *opener*. +The underlying file descriptor for the file object is then obtained +by calling opener with (*name*, *flags*). *opener* must return +an open file descriptor (passing os.open as *opener* results in +functionality similar to passing None). [clinic start generated code]*/ static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=23413f68e6484bbd input=588aac967e0ba74b]*/ +/*[clinic end generated code: output=23413f68e6484bbd input=bac4efcd8f930bf3]*/ { #ifdef MS_WINDOWS wchar_t *widename = NULL; @@ -725,7 +728,6 @@ new_buffersize(fileio *self, size_t currentsize) } /*[clinic input] -@permit_long_docstring_body _io.FileIO.readall cls: defining_class @@ -733,16 +735,18 @@ _io.FileIO.readall Read all data from the file, returned as bytes. -Reads until either there is an error or read() returns size 0 (indicates EOF). -If the file is already at EOF, returns an empty bytes object. +Reads until either there is an error or read() returns size 0 +(indicates EOF). If the file is already at EOF, returns an empty +bytes object. -In non-blocking mode, returns as much data as could be read before EAGAIN. If no -data is available (EAGAIN is returned before bytes are read) returns None. +In non-blocking mode, returns as much data as could be read before +EAGAIN. If no data is available (EAGAIN is returned before bytes +are read) returns None. [clinic start generated code]*/ static PyObject * _io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) -/*[clinic end generated code: output=d546737ec895c462 input=cecda40bf9961299]*/ +/*[clinic end generated code: output=d546737ec895c462 input=65d05bd0169f2df5]*/ { Py_off_t pos, end; PyBytesWriter *writer; @@ -850,7 +854,6 @@ _io_FileIO_readall_impl(fileio *self, PyTypeObject *cls) } /*[clinic input] -@permit_long_docstring_body _io.FileIO.read cls: defining_class size: Py_ssize_t(accept={int, NoneType}) = -1 @@ -858,19 +861,19 @@ _io.FileIO.read Read at most size bytes, returned as bytes. -If size is less than 0, read all bytes in the file making multiple read calls. -See ``FileIO.readall``. +If size is less than 0, read all bytes in the file making multiple +read calls. See ``FileIO.readall``. -Attempts to make only one system call, retrying only per PEP 475 (EINTR). This -means less data may be returned than requested. +Attempts to make only one system call, retrying only per PEP 475 +(EINTR). This means less data may be returned than requested. -In non-blocking mode, returns None if no data is available. Return an empty -bytes object at EOF. +In non-blocking mode, returns None if no data is available. Return +an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) -/*[clinic end generated code: output=bbd749c7c224143e input=752d1ad3db8564a5]*/ +/*[clinic end generated code: output=bbd749c7c224143e input=c7baa3b440af9337]*/ { if (self->fd < 0) return err_closed(); @@ -916,13 +919,13 @@ _io.FileIO.write Write buffer b to file, return number of bytes written. Only makes one system call, so not all of the data may be written. -The number of bytes actually written is returned. In non-blocking mode, -returns None if the write would block. +The number of bytes actually written is returned. In non-blocking +mode, returns None if the write would block. [clinic start generated code]*/ static PyObject * _io_FileIO_write_impl(fileio *self, PyTypeObject *cls, Py_buffer *b) -/*[clinic end generated code: output=927e25be80f3b77b input=2776314f043088f5]*/ +/*[clinic end generated code: output=927e25be80f3b77b input=233f1f70f9e8b09e]*/ { Py_ssize_t n; int err; @@ -1016,7 +1019,6 @@ portable_lseek(fileio *self, PyObject *posobj, int whence, bool suppress_pipe_er } /*[clinic input] -@permit_long_docstring_body _io.FileIO.seek pos: object whence: int = 0 @@ -1024,18 +1026,19 @@ _io.FileIO.seek Move to new file position and return the file position. -Argument offset is a byte count. Optional argument whence defaults to -SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values -are SEEK_CUR or 1 (move relative to current position, positive or negative), -and SEEK_END or 2 (move relative to end of file, usually negative, although -many platforms allow seeking beyond the end of a file). +Argument offset is a byte count. Optional argument whence defaults +to SEEK_SET or 0 (offset from start of file, offset should be >= 0); +other values are SEEK_CUR or 1 (move relative to current position, +positive or negative), and SEEK_END or 2 (move relative to end of +file, usually negative, although many platforms allow seeking beyond +the end of a file). Note that not all file objects are seekable. [clinic start generated code]*/ static PyObject * _io_FileIO_seek_impl(fileio *self, PyObject *pos, int whence) -/*[clinic end generated code: output=c976acdf054e6655 input=f077c492a84c9e62]*/ +/*[clinic end generated code: output=c976acdf054e6655 input=f165a1b4f5d494ad]*/ { if (self->fd < 0) return err_closed(); @@ -1063,6 +1066,7 @@ _io_FileIO_tell_impl(fileio *self) #ifdef HAVE_FTRUNCATE /*[clinic input] +@permit_long_summary _io.FileIO.truncate cls: defining_class size as posobj: object = None @@ -1076,7 +1080,7 @@ The current file position is changed to the value of size. static PyObject * _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj) -/*[clinic end generated code: output=d936732a49e8d5a2 input=c367fb45d6bb2c18]*/ +/*[clinic end generated code: output=d936732a49e8d5a2 input=8f22152bcf900ed2]*/ { Py_off_t pos; int ret; diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index f036ea503b11e86..1253f124108bdbf 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -93,7 +93,6 @@ iobase_unsupported(_PyIO_State *state, const char *message) /* Positioning */ /*[clinic input] -@permit_long_docstring_body _io._IOBase.seek cls: defining_class offset: int(unused=True) @@ -104,11 +103,13 @@ _io._IOBase.seek Change the stream position to the given byte offset. -The offset is interpreted relative to the position indicated by whence. -Values for whence are: +The offset is interpreted relative to the position indicated by +whence. Values for whence are: -* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive -* os.SEEK_CUR or 1 -- current stream position; offset may be negative +* os.SEEK_SET or 0 -- start of stream (the default); offset should + be zero or positive +* os.SEEK_CUR or 1 -- current stream position; offset may be + negative * os.SEEK_END or 2 -- end of stream; offset is usually negative Return the new absolute position. @@ -117,7 +118,7 @@ Return the new absolute position. static PyObject * _io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(offset), int Py_UNUSED(whence)) -/*[clinic end generated code: output=8bd74ea6538ded53 input=a21b5aad416ff6a9]*/ +/*[clinic end generated code: output=8bd74ea6538ded53 input=22eaf07a7a0ee289]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "seek"); @@ -144,14 +145,14 @@ _io._IOBase.truncate Truncate file to size bytes. -File pointer is left unchanged. Size defaults to the current IO position -as reported by tell(). Return the new size. +File pointer is left unchanged. Size defaults to the current IO +position as reported by tell(). Return the new size. [clinic start generated code]*/ static PyObject * _io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *Py_UNUSED(size)) -/*[clinic end generated code: output=2013179bff1fe8ef input=660ac20936612c27]*/ +/*[clinic end generated code: output=2013179bff1fe8ef input=5b3b6ab3c7abd806]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "truncate"); diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 5debae5b42480b8..b8601383ad0a26f 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -225,7 +225,9 @@ write_str(stringio *self, PyObject *obj) if (self->state == STATE_ACCUMULATING) { if (self->string_size == self->pos) { - if (PyUnicodeWriter_WriteStr(self->writer, decoded)) + // gh-149046: Avoid PyUnicodeWriter_WriteStr() which calls str(obj) + // on str subclasses + if (_PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)self->writer, decoded)) goto fail; goto success; } @@ -497,7 +499,8 @@ _io.StringIO.seek Change stream position. -Seek to character offset pos relative to position indicated by whence: +Seek to character offset pos relative to position indicated by +whence: 0 Start of stream (the default). pos should be >= 0; 1 Current position - pos must be 0; 2 End of stream - pos must be 0. @@ -506,7 +509,7 @@ Returns the new absolute position. static PyObject * _io_StringIO_seek_impl(stringio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=c75ced09343a00d7]*/ +/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=ffef24668fd71a5d]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 347bfe976619e89..5b2a20a30c28cb2 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -53,19 +53,19 @@ _unsupported(_PyIO_State *state, const char *message) } /*[clinic input] -@permit_long_docstring_body _io._TextIOBase.detach cls: defining_class / Separate the underlying buffer from the TextIOBase and return it. -After the underlying buffer has been detached, the TextIO is in an unusable state. +After the underlying buffer has been detached, the TextIO is in +an unusable state. [clinic start generated code]*/ static PyObject * _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=50915f40c609eaa4 input=8cd0652c17d7f015]*/ +/*[clinic end generated code: output=50915f40c609eaa4 input=8099c088abcb87d8]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "detach"); @@ -79,14 +79,14 @@ _io._TextIOBase.read Read at most size characters from stream. -Read from underlying buffer until we have size characters or we hit EOF. -If size is negative or omitted, read until EOF. +Read from underlying buffer until we have size characters or we hit +EOF. If size is negative or omitted, read until EOF. [clinic start generated code]*/ static PyObject * _io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(size)) -/*[clinic end generated code: output=51a5178a309ce647 input=f5e37720f9fc563f]*/ +/*[clinic end generated code: output=51a5178a309ce647 input=c9fd4cc1cf1b4614]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "read"); @@ -672,6 +672,8 @@ struct textio int ok; /* initialized? */ int detached; Py_ssize_t chunk_size; + /* Use helpers buffer_*() functions to access buffer; many operations can set it to + NULL (see gh-143008, gh-142594). */ PyObject *buffer; PyObject *encoding; PyObject *encoder; @@ -729,6 +731,67 @@ struct textio #define textio_CAST(op) ((textio *)(op)) +/* Helpers to safely operate on self->buffer. + + self->buffer can be detached (set to NULL) by any user code that is called + leading to NULL pointer dereferences (see gh-143008, gh-142594). Protect + against that by using helpers to check self->buffer validity at callsites. */ +static PyObject * +buffer_access_safe(textio *self) +{ + /* Check self->buffer directly but match errors of CHECK_ATTACHED since this + is called during construction and finalization where self->ok == 0. */ + if (self->buffer == NULL) { + if (self->ok <= 0) { + PyErr_SetString(PyExc_ValueError, + "I/O operation on uninitialized object"); + } + else { + PyErr_SetString(PyExc_ValueError, + "underlying buffer has been detached"); + } + return NULL; + } + + /* Returning a borrowed reference is safe since TextIOWrapper methods are + protected by critical sections. */ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + return self->buffer; +} + +static PyObject * +buffer_getattr(textio *self, PyObject *attr_name) +{ + PyObject *buffer = buffer_access_safe(self); + if (buffer == NULL) { + return NULL; + } + + return PyObject_GetAttr(buffer, attr_name); +} + +static PyObject * +buffer_callmethod_noargs(textio *self, PyObject *name) +{ + PyObject *buffer = buffer_access_safe(self); + if (buffer == NULL) { + return NULL; + } + + return PyObject_CallMethodNoArgs(buffer, name); +} + +static PyObject * +buffer_callmethod_onearg(textio *self, PyObject *name, PyObject *arg) +{ + PyObject *buffer = buffer_access_safe(self); + if (buffer == NULL) { + return NULL; + } + + return PyObject_CallMethodOneArg(buffer, name, arg); +} + static void textiowrapper_set_decoded_chars(textio *self, PyObject *chars); @@ -898,7 +961,7 @@ _textiowrapper_set_decoder(textio *self, PyObject *codec_info, PyObject *res; int r; - res = PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(readable)); + res = buffer_callmethod_noargs(self, &_Py_ID(readable)); if (res == NULL) return -1; @@ -954,7 +1017,7 @@ _textiowrapper_set_encoder(textio *self, PyObject *codec_info, PyObject *res; int r; - res = PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(writable)); + res = buffer_callmethod_noargs(self, &_Py_ID(writable)); if (res == NULL) return -1; @@ -1000,8 +1063,7 @@ _textiowrapper_fix_encoder_state(textio *self) self->encoding_start_of_stream = 1; - PyObject *cookieObj = PyObject_CallMethodNoArgs( - self->buffer, &_Py_ID(tell)); + PyObject *cookieObj = buffer_callmethod_noargs(self, &_Py_ID(tell)); if (cookieObj == NULL) { return -1; } @@ -1061,6 +1123,7 @@ io_check_errors(PyObject *errors) /*[clinic input] +@critical_section _io.TextIOWrapper.__init__ buffer: object encoding: str(accept={str, NoneType}) = None @@ -1104,7 +1167,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, const char *encoding, PyObject *errors, const char *newline, int line_buffering, int write_through) -/*[clinic end generated code: output=72267c0c01032ed2 input=e6cfaaaf6059d4f5]*/ +/*[clinic end generated code: output=72267c0c01032ed2 input=0f077220214c40a4]*/ { PyObject *raw, *codec_info = NULL; PyObject *res; @@ -1568,11 +1631,14 @@ _io_TextIOWrapper_detach_impl(textio *self) /*[clinic end generated code: output=7ba3715cd032d5f2 input=c908a3b4ef203b0f]*/ { PyObject *buffer; - CHECK_ATTACHED(self); if (_PyFile_Flush((PyObject *)self) < 0) { return NULL; } - buffer = self->buffer; + /* _PyFile_Flush could detach before returning; raise an exception. */ + buffer = buffer_access_safe(self); + if (buffer == NULL) { + return NULL; + } self->buffer = NULL; self->detached = 1; return buffer; @@ -1641,7 +1707,7 @@ _textiowrapper_writeflush(textio *self) PyObject *ret; do { - ret = PyObject_CallMethodOneArg(self->buffer, &_Py_ID(write), b); + ret = buffer_callmethod_onearg(self, &_Py_ID(write), b); } while (ret == NULL && _PyIO_trap_eintr()); Py_DECREF(b); // NOTE: We cleared buffer but we don't know how many bytes are actually written @@ -1754,32 +1820,38 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) } } - if (self->pending_bytes == NULL) { - assert(self->pending_bytes_count == 0); - self->pending_bytes = b; - } - else if (!PyList_CheckExact(self->pending_bytes)) { - PyObject *list = PyList_New(2); - if (list == NULL) { + if (bytes_len > 0) { + if (self->pending_bytes == NULL) { + assert(self->pending_bytes_count == 0); + self->pending_bytes = b; + } + else if (!PyList_CheckExact(self->pending_bytes)) { + PyObject *list = PyList_New(2); + if (list == NULL) { + Py_DECREF(b); + return NULL; + } + // Since Python 3.12, allocating GC object won't trigger GC and release + // GIL. See https://github.com/python/cpython/issues/97922 + assert(!PyList_CheckExact(self->pending_bytes)); + PyList_SET_ITEM(list, 0, self->pending_bytes); + PyList_SET_ITEM(list, 1, b); + self->pending_bytes = list; + } + else { + if (PyList_Append(self->pending_bytes, b) < 0) { + Py_DECREF(b); + return NULL; + } Py_DECREF(b); - return NULL; } - // Since Python 3.12, allocating GC object won't trigger GC and release - // GIL. See https://github.com/python/cpython/issues/97922 - assert(!PyList_CheckExact(self->pending_bytes)); - PyList_SET_ITEM(list, 0, self->pending_bytes); - PyList_SET_ITEM(list, 1, b); - self->pending_bytes = list; + + self->pending_bytes_count += bytes_len; } else { - if (PyList_Append(self->pending_bytes, b) < 0) { - Py_DECREF(b); - return NULL; - } Py_DECREF(b); } - self->pending_bytes_count += bytes_len; if (self->pending_bytes_count >= self->chunk_size || needflush || text_needflush) { if (_textiowrapper_writeflush(self) < 0) @@ -1787,7 +1859,8 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) } if (needflush) { - if (_PyFile_Flush(self->buffer) < 0) { + PyObject *buffer = buffer_access_safe(self); + if (buffer == NULL || _PyFile_Flush(buffer) < 0) { return NULL; } } @@ -1917,9 +1990,10 @@ textiowrapper_read_chunk(textio *self, Py_ssize_t size_hint) if (chunk_size == NULL) goto fail; - input_chunk = PyObject_CallMethodOneArg(self->buffer, - (self->has_read1 ? &_Py_ID(read1): &_Py_ID(read)), - chunk_size); + input_chunk = buffer_callmethod_onearg(self, + (self->has_read1 ? &_Py_ID(read1) : + &_Py_ID(read)), + chunk_size); Py_DECREF(chunk_size); if (input_chunk == NULL) goto fail; @@ -2003,7 +2077,7 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (n < 0) { /* Read everything */ - PyObject *bytes = PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(read)); + PyObject *bytes = buffer_callmethod_noargs(self, &_Py_ID(read)); PyObject *decoded; if (bytes == NULL) goto fail; @@ -2600,7 +2674,11 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) Py_DECREF(res); } - res = _PyObject_CallMethod(self->buffer, &_Py_ID(seek), "ii", 0, 2); + PyObject *buf = buffer_access_safe(self); + if (buf == NULL) { + goto fail; + } + res = _PyObject_CallMethod(buf, &_Py_ID(seek), "ii", 0, 2); Py_CLEAR(cookieObj); if (res == NULL) goto fail; @@ -2648,7 +2726,7 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) posobj = PyLong_FromOff_t(cookie.start_pos); if (posobj == NULL) goto fail; - res = PyObject_CallMethodOneArg(self->buffer, &_Py_ID(seek), posobj); + res = buffer_callmethod_onearg(self, &_Py_ID(seek), posobj); Py_DECREF(posobj); if (res == NULL) goto fail; @@ -2665,8 +2743,15 @@ _io_TextIOWrapper_seek_impl(textio *self, PyObject *cookieObj, int whence) if (cookie.chars_to_skip) { /* Just like _read_chunk, feed the decoder and save a snapshot. */ - PyObject *input_chunk = _PyObject_CallMethod(self->buffer, &_Py_ID(read), - "i", cookie.bytes_to_feed); + PyObject *bytes_to_feed = PyLong_FromLong(cookie.bytes_to_feed); + if (bytes_to_feed == NULL) { + goto fail; + } + PyObject *input_chunk = buffer_callmethod_onearg(self, + &_Py_ID(read), + bytes_to_feed); + Py_DECREF(bytes_to_feed); + PyObject *decoded; if (input_chunk == NULL) @@ -2727,13 +2812,13 @@ _io.TextIOWrapper.tell Return the stream position as an opaque number. -The return value of tell() can be given as input to seek(), to restore a -previous stream position. +The return value of tell() can be given as input to seek(), to +restore a previous stream position. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_tell_impl(textio *self) -/*[clinic end generated code: output=4f168c08bf34ad5f input=415d6b4e4f8e6e8c]*/ +/*[clinic end generated code: output=4f168c08bf34ad5f input=aeece020f747fd92]*/ { PyObject *res; PyObject *posobj = NULL; @@ -2765,7 +2850,7 @@ _io_TextIOWrapper_tell_impl(textio *self) goto fail; } - posobj = PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(tell)); + posobj = buffer_callmethod_noargs(self, &_Py_ID(tell)); if (posobj == NULL) goto fail; @@ -2975,7 +3060,7 @@ _io_TextIOWrapper_truncate_impl(textio *self, PyObject *pos) return NULL; } - return PyObject_CallMethodOneArg(self->buffer, &_Py_ID(truncate), pos); + return buffer_callmethod_onearg(self, &_Py_ID(truncate), pos); } static PyObject * @@ -3057,8 +3142,7 @@ static PyObject * _io_TextIOWrapper_fileno_impl(textio *self) /*[clinic end generated code: output=21490a4c3da13e6c input=515e1196aceb97ab]*/ { - CHECK_ATTACHED(self); - return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(fileno)); + return buffer_callmethod_noargs(self, &_Py_ID(fileno)); } /*[clinic input] @@ -3070,8 +3154,7 @@ static PyObject * _io_TextIOWrapper_seekable_impl(textio *self) /*[clinic end generated code: output=ab223dbbcffc0f00 input=71c4c092736c549b]*/ { - CHECK_ATTACHED(self); - return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(seekable)); + return buffer_callmethod_noargs(self, &_Py_ID(seekable)); } /*[clinic input] @@ -3083,8 +3166,7 @@ static PyObject * _io_TextIOWrapper_readable_impl(textio *self) /*[clinic end generated code: output=72ff7ba289a8a91b input=80438d1f01b0a89b]*/ { - CHECK_ATTACHED(self); - return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(readable)); + return buffer_callmethod_noargs(self, &_Py_ID(readable)); } /*[clinic input] @@ -3096,8 +3178,7 @@ static PyObject * _io_TextIOWrapper_writable_impl(textio *self) /*[clinic end generated code: output=a728c71790d03200 input=9d6c22befb0c340a]*/ { - CHECK_ATTACHED(self); - return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(writable)); + return buffer_callmethod_noargs(self, &_Py_ID(writable)); } /*[clinic input] @@ -3109,8 +3190,7 @@ static PyObject * _io_TextIOWrapper_isatty_impl(textio *self) /*[clinic end generated code: output=12be1a35bace882e input=7f83ff04d4d1733d]*/ { - CHECK_ATTACHED(self); - return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(isatty)); + return buffer_callmethod_noargs(self, &_Py_ID(isatty)); } /*[clinic input] @@ -3127,7 +3207,7 @@ _io_TextIOWrapper_flush_impl(textio *self) self->telling = self->seekable; if (_textiowrapper_writeflush(self) < 0) return NULL; - return PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(flush)); + return buffer_callmethod_noargs(self, &_Py_ID(flush)); } /*[clinic input] @@ -3160,8 +3240,9 @@ _io_TextIOWrapper_close_impl(textio *self) else { PyObject *exc = NULL; if (self->finalizing) { - res = PyObject_CallMethodOneArg(self->buffer, &_Py_ID(_dealloc_warn), - (PyObject *)self); + res = buffer_callmethod_onearg(self, + &_Py_ID(_dealloc_warn), + (PyObject *)self); if (res) { Py_DECREF(res); } @@ -3173,7 +3254,7 @@ _io_TextIOWrapper_close_impl(textio *self) exc = PyErr_GetRaisedException(); } - res = PyObject_CallMethodNoArgs(self->buffer, &_Py_ID(close)); + res = buffer_callmethod_noargs(self, &_Py_ID(close)); if (exc != NULL) { _PyErr_ChainExceptions1(exc); Py_CLEAR(res); @@ -3241,8 +3322,7 @@ static PyObject * _io_TextIOWrapper_name_get_impl(textio *self) /*[clinic end generated code: output=8c2f1d6d8756af40 input=26ecec9b39e30e07]*/ { - CHECK_ATTACHED(self); - return PyObject_GetAttr(self->buffer, &_Py_ID(name)); + return buffer_getattr(self, &_Py_ID(name)); } /*[clinic input] @@ -3255,8 +3335,17 @@ static PyObject * _io_TextIOWrapper_closed_get_impl(textio *self) /*[clinic end generated code: output=b49b68f443a85e3c input=7dfcf43f63c7003d]*/ { - CHECK_ATTACHED(self); - return PyObject_GetAttr(self->buffer, &_Py_ID(closed)); + /* If partially constructed or deconstructed, return that the underlying + buffer is closed. + + The code managing the transition is responsible for closing. The closed + attribute is often called in re-initalization, as part of repr in error + cases, and when the I/O stack is garbage collected. */ + if (self->ok <= 0) { + Py_RETURN_TRUE; + } + + return buffer_getattr(self, &_Py_ID(closed)); } /*[clinic input] diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 677d7e85d4e626f..4cd71094e8f459e 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -315,16 +315,16 @@ _io._WindowsConsoleIO.__init__ Open a console buffer by file descriptor. -The mode can be 'rb' (default), or 'wb' for reading or writing bytes. All -other mode characters will be ignored. Mode 'b' will be assumed if it is -omitted. The *opener* parameter is always ignored. +The mode can be 'rb' (default), or 'wb' for reading or writing +bytes. All other mode characters will be ignored. Mode 'b' will be +assumed if it is omitted. The *opener* parameter is always ignored. [clinic start generated code]*/ static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=3fd9cbcdd8d95429 input=7a3eed6bbe998fd9]*/ +/*[clinic end generated code: output=3fd9cbcdd8d95429 input=f31100e2cd724617]*/ { const char *s; wchar_t *name = NULL; @@ -673,12 +673,13 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { maxlen += 1; Py_BLOCK_THREADS newbuf = (wchar_t*)PyMem_Realloc(buf, maxlen * sizeof(wchar_t)); - Py_UNBLOCK_THREADS if (!newbuf) { sig = -1; PyErr_NoMemory(); + Py_UNBLOCK_THREADS break; } + Py_UNBLOCK_THREADS buf = newbuf; /* Only advance by n and not BUFSIZ in this case */ off += n; diff --git a/Modules/_json.c b/Modules/_json.c index 1f454768355cc0c..b057b56b2f9f8d9 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -11,6 +11,8 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST() +#include "pycore_dict.h" // _PyDict_SetItem_Take2() +#include "pycore_list.h" // _PyList_AppendTakeRef() #include "pycore_global_strings.h" // _Py_ID() #include "pycore_pyerrors.h" // _PyErr_FormatNote #include "pycore_runtime.h" // _PyRuntime @@ -657,14 +659,14 @@ JSON string. Unescapes all valid JSON string escape sequences and raises ValueError on attempt to decode an invalid string. If strict is False then literal control characters are allowed in the string. -Returns a tuple of the decoded string and the index of the character in s -after the end quote. +Returns a tuple of the decoded string and the index of the character in +s after the end quote. [clinic start generated code]*/ static PyObject * py_scanstring_impl(PyObject *module, PyObject *pystr, Py_ssize_t end, int strict) -/*[clinic end generated code: output=961740cfae07cdb3 input=cff59e47498f4d8e]*/ +/*[clinic end generated code: output=961740cfae07cdb3 input=6d5abb5947ccc297]*/ { Py_ssize_t next_end = -1; PyObject *rval = scanstring_unicode(pystr, end, strict, &next_end); @@ -752,7 +754,6 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss const void *str; int kind; Py_ssize_t end_idx; - PyObject *val = NULL; PyObject *rval = NULL; PyObject *key = NULL; int has_pairs_hook = (s->object_pairs_hook != Py_None); @@ -802,13 +803,16 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* read any JSON term */ - val = scan_once_unicode(s, memo, pystr, idx, &next_idx); + PyObject *val = scan_once_unicode(s, memo, pystr, idx, &next_idx); if (val == NULL) goto bail; + /* The steal below takes our references to both key and val + (releasing them on failure). Only key is reset for the bail + path; val is never live there, so it needs no cleanup. */ if (has_pairs_hook) { PyObject *item = _PyTuple_FromPairSteal(key, val); - key = val = NULL; + key = NULL; if (item == NULL) goto bail; if (PyList_Append(rval, item) == -1) { @@ -818,10 +822,10 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss Py_DECREF(item); } else { - if (PyDict_SetItem(rval, key, val) < 0) + int err = _PyDict_SetItem_Take2((PyDictObject *)rval, key, val); + key = NULL; + if (err < 0) goto bail; - Py_CLEAR(key); - Py_CLEAR(val); } idx = next_idx; @@ -851,21 +855,20 @@ _parse_object_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ss *next_idx_ptr = idx + 1; if (has_pairs_hook) { - val = PyObject_CallOneArg(s->object_pairs_hook, rval); + PyObject *res = PyObject_CallOneArg(s->object_pairs_hook, rval); Py_DECREF(rval); - return val; + return res; } /* if object_hook is not None: rval = object_hook(rval) */ if (s->object_hook != Py_None) { - val = PyObject_CallOneArg(s->object_hook, rval); + PyObject *res = PyObject_CallOneArg(s->object_hook, rval); Py_DECREF(rval); - return val; + return res; } return rval; bail: Py_XDECREF(key); - Py_XDECREF(val); Py_XDECREF(rval); return NULL; } @@ -882,7 +885,6 @@ _parse_array_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssi const void *str; int kind; Py_ssize_t end_idx; - PyObject *val = NULL; PyObject *rval; Py_ssize_t next_idx; Py_ssize_t comma_idx; @@ -903,14 +905,12 @@ _parse_array_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssi while (1) { /* read any JSON term */ - val = scan_once_unicode(s, memo, pystr, idx, &next_idx); + PyObject *val = scan_once_unicode(s, memo, pystr, idx, &next_idx); if (val == NULL) goto bail; - if (PyList_Append(rval, val) == -1) + if (_PyList_AppendTakeRef((PyListObject *)rval, val) < 0) goto bail; - - Py_CLEAR(val); idx = next_idx; /* skip whitespace between term and , */ @@ -944,13 +944,12 @@ _parse_array_unicode(PyScannerObject *s, PyObject *memo, PyObject *pystr, Py_ssi *next_idx_ptr = idx + 1; /* if array_hook is not None: return array_hook(rval) */ if (!Py_IsNone(s->array_hook)) { - val = PyObject_CallOneArg(s->array_hook, rval); + PyObject *res = PyObject_CallOneArg(s->array_hook, rval); Py_DECREF(rval); - return val; + return res; } return rval; bail: - Py_XDECREF(val); Py_DECREF(rval); return NULL; } diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index 00ee68dcea2d0d9..4335a8bb162414d 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -26,6 +26,19 @@ #error "The maximum block size accepted by liblzma is SIZE_MAX." #endif + +/* + * If the lzma.h we're building against is so old as not to define these, this + * provides their equivalent values so that the names remain defined in Python + * regardless of the header versions used at build time. + */ +#ifndef LZMA_FILTER_ARM64 +#define LZMA_FILTER_ARM64 LZMA_VLI_C(0x0A) +#endif +#ifndef LZMA_FILTER_RISCV +#define LZMA_FILTER_RISCV LZMA_VLI_C(0x0B) +#endif + /* On success, return value >= 0 On failure, return -1 */ static inline Py_ssize_t @@ -373,6 +386,8 @@ lzma_filter_converter(_lzma_state *state, PyObject *spec, void *ptr) case LZMA_FILTER_ARM: case LZMA_FILTER_ARMTHUMB: case LZMA_FILTER_SPARC: + case LZMA_FILTER_ARM64: + case LZMA_FILTER_RISCV: f->options = parse_filter_spec_bcj(state, spec); return f->options != NULL; default: @@ -491,7 +506,9 @@ build_filter_spec(const lzma_filter *f) case LZMA_FILTER_IA64: case LZMA_FILTER_ARM: case LZMA_FILTER_ARMTHUMB: - case LZMA_FILTER_SPARC: { + case LZMA_FILTER_SPARC: + case LZMA_FILTER_ARM64: + case LZMA_FILTER_RISCV: { lzma_options_bcj *options = f->options; if (options) { ADD_FIELD(options, start_offset); @@ -1106,7 +1123,6 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) } /*[clinic input] -@permit_long_docstring_body _lzma.LZMADecompressor.decompress data: Py_buffer @@ -1114,24 +1130,25 @@ _lzma.LZMADecompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=ef4e20ec7122241d input=d5cbd45801b4b8b0]*/ +/*[clinic end generated code: output=ef4e20ec7122241d input=0eb62669c4315dee]*/ { PyObject *result = NULL; @@ -1544,6 +1561,8 @@ lzma_exec(PyObject *module) ADD_INT_PREFIX_MACRO(module, FILTER_ARMTHUMB); ADD_INT_PREFIX_MACRO(module, FILTER_SPARC); ADD_INT_PREFIX_MACRO(module, FILTER_POWERPC); + ADD_INT_PREFIX_MACRO(module, FILTER_ARM64); + ADD_INT_PREFIX_MACRO(module, FILTER_RISCV); ADD_INT_PREFIX_MACRO(module, MF_HC3); ADD_INT_PREFIX_MACRO(module, MF_HC4); ADD_INT_PREFIX_MACRO(module, MF_BT2); diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index a545ff4d80f067d..a4d7273aea718ae 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -50,9 +50,9 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "\n" "Remove a shared memory object (similar to unlink()).\n" "\n" -"Remove a shared memory object name, and, once all processes have unmapped\n" -"the object, de-allocates and destroys the contents of the associated memory\n" -"region."); +"Remove a shared memory object name, and, once all processes have\n" +"unmapped the object, de-allocates and destroys the contents of the\n" +"associated memory region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ {"shm_unlink", (PyCFunction)_posixshmem_shm_unlink, METH_O, _posixshmem_shm_unlink__doc__}, @@ -86,4 +86,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *arg) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=74588a5abba6e36c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e69afacce7b0595e input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index ab45e4136c7d46e..22b4af212662b3c 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -81,15 +81,15 @@ _posixshmem.shm_unlink Remove a shared memory object (similar to unlink()). -Remove a shared memory object name, and, once all processes have unmapped -the object, de-allocates and destroys the contents of the associated memory -region. +Remove a shared memory object name, and, once all processes have +unmapped the object, de-allocates and destroys the contents of the +associated memory region. [clinic start generated code]*/ static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=42f8b23d134b9ff5 input=298369d013dcad63]*/ +/*[clinic end generated code: output=42f8b23d134b9ff5 input=cf7a30ec6503cf78]*/ { int rv; int async_err = 0; diff --git a/Modules/_opcode.c b/Modules/_opcode.c index dedf17f76dfc9b7..2a34559fd1f4378 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -119,7 +119,7 @@ _opcode_has_const_impl(PyObject *module, int opcode) } /*[clinic input] - +@permit_long_summary _opcode.has_name -> bool opcode: int @@ -129,7 +129,7 @@ Return True if the opcode accesses an attribute by name, False otherwise. static int _opcode_has_name_impl(PyObject *module, int opcode) -/*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ +/*[clinic end generated code: output=b49a83555c2fa517 input=8faf669024d97fad]*/ { return IS_VALID_OPCODE(opcode) && OPCODE_HAS_NAME(opcode); } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 9874f9475ac0296..034dd06b6ab4609 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -12,6 +12,7 @@ #include "pycore_bytesobject.h" // _PyBytesWriter #include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() +#include "pycore_dict.h" // _PyDict_SetItem_Take2() #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyNone_Type @@ -2055,22 +2056,34 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * return NULL; } if (PyDict_CheckExact(modules)) { + PyObject *found_name = NULL; + int error = 0; i = 0; + Py_BEGIN_CRITICAL_SECTION(modules); while (PyDict_Next(modules, &i, &module_name, &module)) { Py_INCREF(module_name); Py_INCREF(module); if (_checkmodule(module_name, module, global, dotted_path) == 0) { Py_DECREF(module); - Py_DECREF(modules); - return module_name; + found_name = module_name; + break; } Py_DECREF(module); Py_DECREF(module_name); if (PyErr_Occurred()) { - Py_DECREF(modules); - return NULL; + error = 1; + break; } } + Py_END_CRITICAL_SECTION(); + if (error) { + Py_DECREF(modules); + return NULL; + } + if (found_name != NULL) { + Py_DECREF(modules); + return found_name; + } } else { PyObject *iterator = PyObject_GetIter(modules); @@ -3179,7 +3192,7 @@ static int batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *item = NULL; - Py_ssize_t this_batch, total; + Py_ssize_t this_batch, total, list_size; const char append_op = APPEND; const char appends_op = APPENDS; @@ -3188,14 +3201,18 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) assert(obj != NULL); assert(self->proto > 0); assert(PyList_CheckExact(obj)); - assert(PyList_GET_SIZE(obj)); + + list_size = PyList_GET_SIZE(obj); /* Write in batches of BATCHSIZE. */ total = 0; do { - if (PyList_GET_SIZE(obj) - total == 1) { - item = PyList_GET_ITEM(obj, total); - Py_INCREF(item); + if (list_size - total == 1) { + item = PyList_GetItemRef(obj, total); + if (item == NULL) { + _PyErr_FormatNote("when serializing %T item %zd", obj, total); + return -1; + } int err = save(state, self, item, 0); Py_DECREF(item); if (err < 0) { @@ -3210,8 +3227,11 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; while (total < PyList_GET_SIZE(obj)) { - item = PyList_GET_ITEM(obj, total); - Py_INCREF(item); + item = PyList_GetItemRef(obj, total); + if (item == NULL) { + _PyErr_FormatNote("when serializing %T item %zd", obj, total); + return -1; + } int err = save(state, self, item, 0); Py_DECREF(item); if (err < 0) { @@ -3224,8 +3244,14 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) } if (_Pickler_Write(self, &appends_op, 1) < 0) return -1; + if (PyList_GET_SIZE(obj) != list_size) { + PyErr_Format( + PyExc_RuntimeError, + "list changed size during iteration"); + return -1; + } - } while (total < PyList_GET_SIZE(obj)); + } while (total < list_size); return 0; } @@ -3452,7 +3478,7 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *or * Note that this currently doesn't work for protocol 0. */ static int -batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) +batch_dict_exact_impl(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *key = NULL, *value = NULL; int i; @@ -3525,6 +3551,18 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) return -1; } +/* gh-146452: Wrap the dict iteration in a critical section to prevent + concurrent mutation from invalidating PyDict_Next() iteration state. */ +static int +batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(obj); + ret = batch_dict_exact_impl(state, self, obj); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int save_dict(PickleState *state, PicklerObject *self, PyObject *obj) { @@ -5110,9 +5148,7 @@ _pickle_PicklerMemoProxy_copy_impl(PicklerMemoProxyObject *self) Py_DECREF(key); goto error; } - status = PyDict_SetItem(new_memo, key, value); - Py_DECREF(key); - Py_DECREF(value); + status = _PyDict_SetItem_Take2((PyDictObject *)new_memo, key, value); if (status < 0) goto error; } diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index b7f39ea3d499e4c..2aa3923f68e66ad 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -63,7 +63,7 @@ # endif #endif -#if defined(__FreeBSD__) || (defined(__APPLE__) && defined(__MACH__)) || defined(__DragonFly__) +#if defined(__CYGWIN__) || defined(__FreeBSD__) || (defined(__APPLE__) && defined(__MACH__)) || defined(__DragonFly__) # define FD_DIR "/dev/fd" #else # define FD_DIR "/proc/self/fd" @@ -962,7 +962,6 @@ do_fork_exec(char *const exec_array[], } /*[clinic input] -@permit_long_docstring_body _posixsubprocess.fork_exec as subprocess_fork_exec args as process_args: object executable_list: object @@ -990,15 +989,15 @@ _posixsubprocess.fork_exec as subprocess_fork_exec Spawn a fresh new child process. -Fork a child process, close parent file descriptors as appropriate in the -child and duplicate the few that are needed before calling exec() in the -child process. +Fork a child process, close parent file descriptors as appropriate in +the child and duplicate the few that are needed before calling exec() in +the child process. -If close_fds is True, close file descriptors 3 and higher, except those listed -in the sorted tuple pass_fds. +If close_fds is True, close file descriptors 3 and higher, except those +listed in the sorted tuple pass_fds. -The preexec_fn, if supplied, will be called immediately before closing file -descriptors and exec. +The preexec_fn, if supplied, will be called immediately before closing +file descriptors and exec. WARNING: preexec_fn is NOT SAFE if your application uses threads. It may trigger infrequent, difficult to debug deadlocks. @@ -1023,7 +1022,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, PyObject *preexec_fn) -/*[clinic end generated code: output=288464dc56e373c7 input=58e0db771686f4f6]*/ +/*[clinic end generated code: output=288464dc56e373c7 input=5e56eac3e036e349]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; @@ -1091,8 +1090,14 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, goto cleanup; } borrowed_arg = PySequence_Fast_GET_ITEM(fast_args, arg_num); - if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) + /* borrowed_arg is only borrowed; its __fspath__() may run Python + that drops fast_args' last reference to it. */ + Py_INCREF(borrowed_arg); + if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) { + Py_DECREF(borrowed_arg); goto cleanup; + } + Py_DECREF(borrowed_arg); PyTuple_SET_ITEM(converted_args, arg_num, converted_arg); } diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index ed925f3525a9a7d..af54e42a6af584c 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -154,8 +154,6 @@ RingBuf_Get(RingBuf *buf) } // Returns 0 on success or -1 if the buffer failed to grow. -// -// Steals a reference to item. static int RingBuf_Put(RingBuf *buf, PyObject *item) { @@ -165,11 +163,10 @@ RingBuf_Put(RingBuf *buf, PyObject *item) // Buffer is full, grow it. if (resize_ringbuf(buf, buf->items_cap * 2) < 0) { PyErr_NoMemory(); - Py_DECREF(item); return -1; } } - buf->items[buf->put_idx] = item; + buf->items[buf->put_idx] = Py_NewRef(item); buf->put_idx = (buf->put_idx + 1) % buf->items_cap; buf->num_items++; return 0; @@ -276,16 +273,13 @@ maybe_handoff_item(void *arg, void *park_arg, int has_more_waiters) { HandoffData *data = (HandoffData*)arg; PyObject **item = (PyObject**)park_arg; - if (item == NULL) { - // No threads were waiting - data->handed_off = false; - } - else { + data->queue->has_threads_waiting = has_more_waiters; + + data->handed_off = item != NULL; + if (data->handed_off) { // There was at least one waiting thread, hand off the item - *item = data->item; - data->handed_off = true; + *item = Py_NewRef(data->item); } - data->queue->has_threads_waiting = has_more_waiters; } /*[clinic input] @@ -297,31 +291,33 @@ _queue.SimpleQueue.put Put the item on the queue. -The optional 'block' and 'timeout' arguments are ignored, as this method -never blocks. They are provided for compatibility with the Queue class. +The optional 'block' and 'timeout' arguments are ignored, as this +method never blocks. They are provided for compatibility with the +Queue class. [clinic start generated code]*/ static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) -/*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ +/*[clinic end generated code: output=4333136e88f90d8b input=9f9ff270a74670c3]*/ { - HandoffData data = { - .handed_off = 0, - .item = Py_NewRef(item), - .queue = self, - }; if (self->has_threads_waiting) { + HandoffData data = { + .handed_off = 0, + .item = item, + .queue = self, + }; // Try to hand the item off directly if there are threads waiting _PyParkingLot_Unpark(&self->has_threads_waiting, maybe_handoff_item, &data); - } - if (!data.handed_off) { - if (RingBuf_Put(&self->buf, item) < 0) { - return NULL; + if (data.handed_off) { + Py_RETURN_NONE; } } + if (RingBuf_Put(&self->buf, item) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -365,10 +361,11 @@ _queue.SimpleQueue.get Remove and return an item from the queue. -If optional args 'block' is true and 'timeout' is None (the default), -block if necessary until an item is available. If 'timeout' is -a non-negative number, it blocks at most 'timeout' seconds and raises -the Empty exception if no item was available within that time. +If optional args 'block' is true and 'timeout' is None (the +default), block if necessary until an item is available. If +'timeout' is a non-negative number, it blocks at most 'timeout' +seconds and raises the Empty exception if no item was available +within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). @@ -378,7 +375,7 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) -/*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ +/*[clinic end generated code: output=5c2cca914cd1e55b input=afa0889bbc6b4761]*/ { PyTime_t endtime = 0; @@ -553,7 +550,7 @@ static PyMethodDef simplequeue_methods[] = { _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF _QUEUE_SIMPLEQUEUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("SimpleQueues are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 0fb734816517485..a06966be23be1ef 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -123,9 +123,9 @@ typedef struct { /*[clinic input] module _random -class _random.Random "RandomObject *" "_randomstate_type(type)->Random_Type" +class _random.Random "RandomObject *" "(PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=70a2c99619474983]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f04bcbfba61a322e]*/ /* Random methods */ @@ -549,27 +549,20 @@ _random_Random_getrandbits_impl(RandomObject *self, uint64_t k) return result; } -static int -random_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - PyObject *arg = NULL; - _randomstate *state = _randomstate_type(Py_TYPE(self)); - - if ((Py_IS_TYPE(self, (PyTypeObject *)state->Random_Type) || - Py_TYPE(self)->tp_init == ((PyTypeObject*)state->Random_Type)->tp_init) && - !_PyArg_NoKeywords("Random", kwds)) { - return -1; - } - - if (PyTuple_GET_SIZE(args) > 1) { - PyErr_SetString(PyExc_TypeError, "Random() requires 0 or 1 argument"); - return -1; - } +/*[clinic input] +@critical_section +@text_signature "($self, [seed])" +_random.Random.__init__ as random_init - if (PyTuple_GET_SIZE(args) == 1) - arg = PyTuple_GET_ITEM(args, 0); + seed: object = NULL + / +[clinic start generated code]*/ - return random_seed(RandomObject_CAST(self), arg); +static int +random_init_impl(RandomObject *self, PyObject *seed) +/*[clinic end generated code: output=260734a3739c394f input=e516bf32e8a05e28]*/ +{ + return random_seed(self, seed); } diff --git a/Modules/_remote_debugging/_remote_debugging.h b/Modules/_remote_debugging/_remote_debugging.h index 7369cd1514c296d..635e6e208902af5 100644 --- a/Modules/_remote_debugging/_remote_debugging.h +++ b/Modules/_remote_debugging/_remote_debugging.h @@ -30,6 +30,7 @@ extern "C" { #include "internal/pycore_llist.h" // struct llist_node #include "internal/pycore_long.h" // _PyLong_GetZero #include "internal/pycore_pyerrors.h" // _PyErr_FormatFromCause +#include "internal/pycore_pyhash.h" // _Py_HashPointerRaw #include "internal/pycore_stackref.h" // Py_TAG_BITS #include "../../Python/remote_debug.h" @@ -179,7 +180,7 @@ typedef enum _WIN32_THREADSTATE { #define set_exception_cause(unwinder, exc_type, message) \ do { \ assert(PyErr_Occurred() && "function returned -1 without setting exception"); \ - if (unwinder->debug) { \ + if (unwinder->debug && !_Py_RemoteDebug_HasPermissionError()) { \ _set_debug_exception_cause(exc_type, message); \ } \ } while (0) @@ -215,6 +216,8 @@ typedef struct { PyObject *file_name; int first_lineno; PyObject *linetable; // bytes + PyObject *last_frame_info; + ptrdiff_t last_addrq; uintptr_t addr_code_adaptive; } CachedCodeMetadata; @@ -224,11 +227,41 @@ typedef struct { typedef struct { uint64_t thread_id; // 0 = empty slot + uintptr_t thread_state_addr; uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; Py_ssize_t num_addrs; + PyObject *thread_id_obj; // owned reference, NULL if empty PyObject *frame_list; // owned reference, NULL if empty } FrameCacheEntry; +#define INTERPRETER_THREAD_CACHE_SIZE 32 +#if (INTERPRETER_THREAD_CACHE_SIZE & (INTERPRETER_THREAD_CACHE_SIZE - 1)) != 0 +# error "INTERPRETER_THREAD_CACHE_SIZE must be a power of two" +#endif + +// The two per-interpreter L2 caches below are split into per-field tables so +// that a writer rebinding one slot cannot leave stale data in a field owned by +// the other when the slot is reused across interpreters. +typedef struct { + uintptr_t interpreter_addr; + uintptr_t thread_state_addr; +} InterpreterTstateCacheEntry; +typedef struct { + uintptr_t interpreter_addr; + uint64_t code_object_generation; +} InterpreterGenerationCacheEntry; + +// Carries already-read thread state and/or frame buffers across helpers so the +// downstream callee can skip a remote read. Address fields are caller-supplied +// inputs; buffer pointers (tstate, frame) are NULL unless a prior batched read +// successfully populated them. +typedef struct { + const char *tstate; + uintptr_t tstate_addr; + const char *frame; + uintptr_t frame_addr; +} RemoteReadPrefetch; + /* Statistics for profiling performance analysis */ typedef struct { uint64_t total_samples; // Total number of get_stack_trace calls @@ -242,14 +275,44 @@ typedef struct { uint64_t code_object_cache_hits; // Code object cache hits uint64_t code_object_cache_misses; // Code object cache misses uint64_t stale_cache_invalidations; // Times stale entries were cleared + uint64_t batched_read_attempts; // Batched remote-read attempts + uint64_t batched_read_successes; // Attempts that read all requested segments + uint64_t batched_read_misses; // Attempts that fell back or partially read + uint64_t batched_read_segments_requested; // Segments requested by batched reads + uint64_t batched_read_segments_completed; // Segments completed by batched reads } UnwinderStats; +#if defined(__GNUC__) || defined(__clang__) +# define REMOTE_DEBUG_UNLIKELY(value) __builtin_expect(!!(value), 0) +#else +# define REMOTE_DEBUG_UNLIKELY(value) (value) +#endif + /* Stats tracking macros - no-op when stats collection is disabled */ #define STATS_INC(unwinder, field) \ - do { if ((unwinder)->collect_stats) (unwinder)->stats.field++; } while(0) + do { if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) (unwinder)->stats.field++; } while(0) #define STATS_ADD(unwinder, field, val) \ - do { if ((unwinder)->collect_stats) (unwinder)->stats.field += (val); } while(0) + do { if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) (unwinder)->stats.field += (val); } while(0) + +#if HAVE_PROCESS_VM_READV +# define STATS_BATCHED_READ(unwinder, requested, completed) \ + do { \ + if (REMOTE_DEBUG_UNLIKELY((unwinder)->collect_stats)) { \ + (unwinder)->stats.batched_read_attempts++; \ + (unwinder)->stats.batched_read_segments_requested += (uint64_t)(requested); \ + (unwinder)->stats.batched_read_segments_completed += (uint64_t)(completed); \ + if ((completed) == (requested)) { \ + (unwinder)->stats.batched_read_successes++; \ + } \ + else { \ + (unwinder)->stats.batched_read_misses++; \ + } \ + } \ + } while(0) +#else +# define STATS_BATCHED_READ(unwinder, requested, completed) ((void)0) +#endif typedef struct { PyTypeObject *RemoteDebugging_Type; @@ -290,7 +353,6 @@ typedef struct { struct _Py_AsyncioModuleDebugOffsets async_debug_offsets; uintptr_t interpreter_addr; uintptr_t tstate_addr; - uint64_t code_object_generation; _Py_hashtable_t *code_object_cache; int debug; int only_active_thread; @@ -302,9 +364,17 @@ typedef struct { int cache_frames; int collect_stats; // whether to collect statistics uint32_t stale_invalidation_counter; // counter for throttling frame_cache_invalidate_stale + // L1 single-entry shortcut over cached_tstates[]: most workloads sample one + // interpreter, so check these pairs before hashing into the table below. + uintptr_t cached_tstate_interpreter_addr; + uintptr_t cached_tstate_addr; + uintptr_t cached_generation_interpreter_addr; + uint64_t cached_code_object_generation; RemoteDebuggingState *cached_state; FrameCacheEntry *frame_cache; // preallocated array of FRAME_CACHE_MAX_THREADS entries UnwinderStats stats; // statistics for performance analysis + InterpreterTstateCacheEntry cached_tstates[INTERPRETER_THREAD_CACHE_SIZE]; + InterpreterGenerationCacheEntry cached_generations[INTERPRETER_THREAD_CACHE_SIZE]; #ifdef Py_GIL_DISABLED uint32_t tlbc_generation; _Py_hashtable_t *tlbc_cache; @@ -361,11 +431,13 @@ typedef struct { typedef struct { /* Inputs */ uintptr_t frame_addr; // Starting frame address + uintptr_t thread_state_addr; // Owning thread state address uintptr_t base_frame_addr; // Sentinel at bottom (for validation) uintptr_t gc_frame; // GC frame address (0 if not tracking) uintptr_t last_profiled_frame; // Last cached frame (0 if no cache) StackChunkList *chunks; // Pre-copied stack chunks int skip_first_frame; // Skip frame_addr itself (continue from its caller) + RemoteReadPrefetch prefetch; // Optional already-read thread/frame buffers /* Outputs */ PyObject *frame_info; // List to append FrameInfo objects @@ -548,6 +620,7 @@ extern int process_frame_chain( extern int frame_cache_init(RemoteUnwinderObject *unwinder); extern void frame_cache_cleanup(RemoteUnwinderObject *unwinder); extern FrameCacheEntry *frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id); +extern FrameCacheEntry *frame_cache_find_by_tstate(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr); extern int clear_last_profiled_frames(RemoteUnwinderObject *unwinder); extern void frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result); extern int frame_cache_lookup_and_extend( @@ -566,6 +639,7 @@ extern int frame_cache_store( PyObject *frame_list, const uintptr_t *addrs, Py_ssize_t num_addrs, + uintptr_t thread_state_addr, uintptr_t base_frame_addr, uintptr_t last_frame_visited); @@ -605,7 +679,8 @@ extern PyObject* unwind_stack_for_thread( uintptr_t *current_tstate, uintptr_t gil_holder_tstate, uintptr_t gc_frame, - uintptr_t main_thread_tstate + uintptr_t main_thread_tstate, + const RemoteReadPrefetch *prefetch ); /* Thread stopping functions (for blocking mode) */ diff --git a/Modules/_remote_debugging/asyncio.c b/Modules/_remote_debugging/asyncio.c index fc7487d4044bfb2..44a9a3cbce0061a 100644 --- a/Modules/_remote_debugging/asyncio.c +++ b/Modules/_remote_debugging/asyncio.c @@ -22,35 +22,38 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio", NULL); if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } } #elif defined(__linux__) && HAVE_PROCESS_VM_READV // On Linux, search for asyncio debug in executable or DLL address = search_linux_map_for_section(handle, "AsyncioDebug", "python", NULL); if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } } #elif defined(__APPLE__) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python address = search_map_for_section(handle, "AsyncioDebug", "libpython", NULL); - if (address == 0) { + if (address == 0 && !_Py_RemoteDebug_HasPermissionError()) { PyErr_Clear(); address = search_map_for_section(handle, "AsyncioDebug", "python", NULL); } if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } } #else Py_UNREACHABLE(); @@ -96,10 +99,12 @@ ensure_async_debug_offsets(RemoteUnwinderObject *unwinder) return -1; } if (result < 0) { - PyErr_Clear(); - PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); - set_exception_cause(unwinder, PyExc_RuntimeError, - "AsyncioDebug section unavailable - asyncio module may not be loaded in target process"); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyErr_Clear(); + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "AsyncioDebug section unavailable - asyncio module may not be loaded in target process"); + } return -1; } @@ -218,7 +223,7 @@ parse_task_name( if ((GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_LONG_SUBCLASS)) { long res = read_py_long(unwinder, task_name_addr); - if (res == -1) { + if (res == -1 && PyErr_Occurred()) { set_exception_cause(unwinder, PyExc_RuntimeError, "Task name PyLong parsing failed"); return NULL; } diff --git a/Modules/_remote_debugging/binary_io.h b/Modules/_remote_debugging/binary_io.h index 18f989f672e1034..d4188335c0b6d0a 100644 --- a/Modules/_remote_debugging/binary_io.h +++ b/Modules/_remote_debugging/binary_io.h @@ -61,11 +61,36 @@ extern "C" { #define HDR_SIZE_COMPRESSION 4 #define FILE_HEADER_SIZE (HDR_OFF_COMPRESSION + HDR_SIZE_COMPRESSION) #define FILE_HEADER_PLACEHOLDER_SIZE 64 -#define SAMPLE_HEADER_FIXED_SIZE (sizeof(uint64_t) + sizeof(uint32_t) + 1) static_assert(FILE_HEADER_SIZE <= FILE_HEADER_PLACEHOLDER_SIZE, "FILE_HEADER_SIZE exceeds FILE_HEADER_PLACEHOLDER_SIZE"); +/* Sample header field offsets and sizes */ +#define SMP_OFF_THREAD_ID 0 +#define SMP_SIZE_THREAD_ID sizeof(uint64_t) +#define SMP_OFF_INTERPRETER_ID (SMP_OFF_THREAD_ID + SMP_SIZE_THREAD_ID) +#define SMP_SIZE_INTERPRETER_ID sizeof(uint32_t) +#define SMP_OFF_ENCODING (SMP_OFF_INTERPRETER_ID + SMP_SIZE_INTERPRETER_ID) +#define SMP_SIZE_ENCODING sizeof(uint8_t) +#define SAMPLE_HEADER_FIXED_SIZE (SMP_OFF_ENCODING + SMP_SIZE_ENCODING) + +static_assert(SAMPLE_HEADER_FIXED_SIZE == 13, + "SAMPLE_HEADER_FIXED_SIZE must remain 13"); + +/* Footer field offsets and sizes */ +#define FTR_OFF_STRINGS 0 +#define FTR_SIZE_STRINGS sizeof(uint32_t) +#define FTR_OFF_FRAMES (FTR_OFF_STRINGS + FTR_SIZE_STRINGS) +#define FTR_SIZE_FRAMES sizeof(uint32_t) +#define FTR_OFF_FILE_SIZE (FTR_OFF_FRAMES + FTR_SIZE_FRAMES) +#define FTR_SIZE_FILE_SIZE sizeof(uint64_t) +#define FTR_OFF_CHECKSUM (FTR_OFF_FILE_SIZE + FTR_SIZE_FILE_SIZE) +#define FTR_SIZE_CHECKSUM (2 * sizeof(uint64_t)) +#define FILE_FOOTER_SIZE (FTR_OFF_CHECKSUM + FTR_SIZE_CHECKSUM) + +static_assert(FILE_FOOTER_SIZE == 32, + "FILE_FOOTER_SIZE must remain 32"); + /* Buffer sizes: 512KB balances syscall amortization against memory use, * and aligns well with filesystem block sizes and zstd dictionary windows */ #define WRITE_BUFFER_SIZE (512 * 1024) @@ -228,7 +253,6 @@ typedef struct { /* Main binary writer structure */ typedef struct { FILE *fp; - char *filename; /* Write buffer for batched I/O */ uint8_t *write_buffer; @@ -286,10 +310,7 @@ typedef struct { /* Main binary reader structure */ typedef struct { - char *filename; - #if USE_MMAP - int fd; uint8_t *mapped_data; size_t mapped_size; #else @@ -497,7 +518,7 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t *capacity, size_t elem_ * Create a new binary writer. * * Arguments: - * filename: Path to output file + * path: Path to output file * sample_interval_us: Sampling interval in microseconds * compression_type: COMPRESSION_NONE or COMPRESSION_ZSTD * start_time_us: Start timestamp in microseconds (from time.monotonic() * 1e6) @@ -506,7 +527,7 @@ grow_array_inplace(void **ptr_addr, size_t count, size_t *capacity, size_t elem_ * New BinaryWriter* on success, NULL on failure (PyErr set) */ BinaryWriter *binary_writer_create( - const char *filename, + PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us @@ -558,12 +579,12 @@ void binary_writer_destroy(BinaryWriter *writer); * Open a binary file for reading. * * Arguments: - * filename: Path to input file + * path: Path to input file * * Returns: * New BinaryReader* on success, NULL on failure (PyErr set) */ -BinaryReader *binary_reader_open(const char *filename); +BinaryReader *binary_reader_open(PyObject *path); /* * Replay samples from binary file through a collector. diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 6c32ef70ac3f659..ce1c3d232c94e0f 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -23,15 +23,11 @@ * ============================================================================ */ /* File structure sizes */ -#define FILE_FOOTER_SIZE 32 #define MIN_DECOMPRESS_BUFFER_SIZE (64 * 1024) /* Minimum decompression buffer */ /* Progress callback frequency */ #define PROGRESS_CALLBACK_INTERVAL 1000 -/* Maximum decompression size limit (1GB) */ -#define MAX_DECOMPRESS_SIZE (1ULL << 30) - /* ============================================================================ * BINARY READER IMPLEMENTATION * ============================================================================ */ @@ -47,8 +43,8 @@ reader_parse_header(BinaryReader *reader, const uint8_t *data, size_t file_size) /* Use memcpy to avoid strict aliasing violations and unaligned access */ uint32_t magic; uint32_t version; - memcpy(&magic, &data[0], sizeof(magic)); - memcpy(&version, &data[4], sizeof(version)); + memcpy(&magic, &data[HDR_OFF_MAGIC], HDR_SIZE_MAGIC); + memcpy(&version, &data[HDR_OFF_VERSION], HDR_SIZE_VERSION); /* Detect endianness from magic number */ if (magic == BINARY_FORMAT_MAGIC) { @@ -119,8 +115,8 @@ reader_parse_footer(BinaryReader *reader, const uint8_t *data, size_t file_size) const uint8_t *footer = data + file_size - FILE_FOOTER_SIZE; /* Use memcpy to avoid strict aliasing violations */ uint32_t strings_count, frames_count; - memcpy(&strings_count, &footer[0], sizeof(strings_count)); - memcpy(&frames_count, &footer[4], sizeof(frames_count)); + memcpy(&strings_count, &footer[FTR_OFF_STRINGS], FTR_SIZE_STRINGS); + memcpy(&frames_count, &footer[FTR_OFF_FRAMES], FTR_SIZE_FRAMES); reader->strings_count = SWAP32_IF(reader->needs_swap, strings_count); reader->frames_count = SWAP32_IF(reader->needs_swap, frames_count); @@ -362,7 +358,7 @@ reader_parse_frame_table(BinaryReader *reader, const uint8_t *data, size_t file_ } BinaryReader * -binary_reader_open(const char *filename) +binary_reader_open(PyObject *path) { BinaryReader *reader = PyMem_Calloc(1, sizeof(BinaryReader)); if (!reader) { @@ -370,32 +366,36 @@ binary_reader_open(const char *filename) return NULL; } -#if USE_MMAP - reader->fd = -1; /* Explicit initialization for cleanup safety */ -#endif - - reader->filename = PyMem_Malloc(strlen(filename) + 1); - if (!reader->filename) { - PyMem_Free(reader); - PyErr_NoMemory(); - return NULL; - } - strcpy(reader->filename, filename); - #if USE_MMAP /* Open with mmap on Unix */ - reader->fd = open(filename, O_RDONLY); - if (reader->fd < 0) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); + FILE *fp = Py_fopen(path, "rb"); + if (!fp) { goto error; } + int fd = fileno(fp); struct stat st; - if (fstat(reader->fd, &st) < 0) { + if (fstat(fd, &st) < 0) { PyErr_SetFromErrno(PyExc_IOError); + Py_fclose(fp); + goto error; + } + if (st.st_size < 0) { + PyErr_SetString(PyExc_IOError, "Invalid negative file size"); + Py_fclose(fp); + goto error; + } + if ((uintmax_t)st.st_size > SIZE_MAX) { + PyErr_SetString(PyExc_OverflowError, "File is too large to map"); + Py_fclose(fp); goto error; } reader->mapped_size = st.st_size; + if (reader->mapped_size == 0) { + PyErr_SetString(PyExc_ValueError, "File too small for header"); + Py_fclose(fp); + goto error; + } /* Map the file into memory. * MAP_POPULATE (Linux-only) pre-faults all pages at mmap time, which: @@ -404,14 +404,15 @@ binary_reader_open(const char *filename) */ #ifdef __linux__ reader->mapped_data = mmap(NULL, reader->mapped_size, PROT_READ, - MAP_PRIVATE | MAP_POPULATE, reader->fd, 0); + MAP_PRIVATE | MAP_POPULATE, fd, 0); #else reader->mapped_data = mmap(NULL, reader->mapped_size, PROT_READ, - MAP_PRIVATE, reader->fd, 0); + MAP_PRIVATE, fd, 0); #endif if (reader->mapped_data == MAP_FAILED) { reader->mapped_data = NULL; PyErr_SetFromErrno(PyExc_IOError); + Py_fclose(fp); goto error; } @@ -432,19 +433,23 @@ binary_reader_open(const char *filename) /* Add file descriptor-level hints for better kernel I/O scheduling */ #if defined(__linux__) && defined(POSIX_FADV_SEQUENTIAL) - (void)posix_fadvise(reader->fd, 0, 0, POSIX_FADV_SEQUENTIAL); + (void)posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); if (reader->mapped_size > (64 * 1024 * 1024)) { - (void)posix_fadvise(reader->fd, 0, 0, POSIX_FADV_WILLNEED); + (void)posix_fadvise(fd, 0, 0, POSIX_FADV_WILLNEED); } #endif + if (Py_fclose(fp) != 0) { + PyErr_SetFromErrno(PyExc_IOError); + goto error; + } + uint8_t *data = reader->mapped_data; size_t file_size = reader->mapped_size; #else /* Use stdio on Windows */ - reader->fp = fopen(filename, "rb"); + reader->fp = Py_fopen(path, "rb"); if (!reader->fp) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); goto error; } @@ -457,7 +462,15 @@ binary_reader_open(const char *filename) PyErr_SetFromErrno(PyExc_IOError); goto error; } + if ((uint64_t)file_size_off > SIZE_MAX) { + PyErr_SetString(PyExc_OverflowError, "File is too large to read"); + goto error; + } reader->file_size = (size_t)file_size_off; + if (reader->file_size == 0) { + PyErr_SetString(PyExc_ValueError, "File too small for header"); + goto error; + } if (FSEEK64(reader->fp, 0, SEEK_SET) != 0) { PyErr_SetFromErrno(PyExc_IOError); goto error; @@ -469,8 +482,18 @@ binary_reader_open(const char *filename) goto error; } - if (fread(reader->file_data, 1, reader->file_size, reader->fp) != reader->file_size) { - PyErr_SetFromErrno(PyExc_IOError); + size_t nread = fread(reader->file_data, 1, reader->file_size, reader->fp); + if (nread != reader->file_size) { + int err = errno; + if (ferror(reader->fp) && err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_IOError); + } + else { + PyErr_Format(PyExc_ValueError, + "Unexpected end of file: read %zu of %zu bytes", + nread, reader->file_size); + } goto error; } @@ -563,6 +586,14 @@ reader_get_or_create_thread_state(BinaryReader *reader, uint64_t thread_id, } } + if (reader->thread_state_count >= reader->thread_count) { + PyErr_Format(PyExc_ValueError, + "Invalid thread count: sample data contains more unique threads than declared in header " + "(declared %u, found at least %zu)", + reader->thread_count, reader->thread_state_count + 1); + return NULL; + } + if (!reader->thread_states) { reader->thread_state_capacity = 16; reader->thread_states = PyMem_Calloc(reader->thread_state_capacity, sizeof(ReaderThreadState)); @@ -785,9 +816,9 @@ build_frame_list(RemoteDebuggingState *state, BinaryReader *reader, if (frame->lineno != LOCATION_NOT_AVAILABLE) { location = Py_BuildValue("(iiii)", frame->lineno, - frame->end_lineno != LOCATION_NOT_AVAILABLE ? frame->end_lineno : frame->lineno, - frame->column != LOCATION_NOT_AVAILABLE ? frame->column : 0, - frame->end_column != LOCATION_NOT_AVAILABLE ? frame->end_column : 0); + frame->end_lineno, + frame->column, + frame->end_column); if (!location) { Py_DECREF(frame_info); goto error; @@ -949,10 +980,16 @@ invoke_progress_callback(PyObject *callback, Py_ssize_t current, uint32_t total) Py_ssize_t binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progress_callback) { - if (!PyObject_HasAttrString(collector, "collect")) { + PyObject *collect_method; + int has_collect = PyObject_GetOptionalAttrString(collector, "collect", &collect_method); + if (has_collect < 0) { + return -1; + } + if (has_collect == 0) { PyErr_SetString(PyExc_TypeError, "Collector must have a collect() method"); return -1; } + Py_DECREF(collect_method); /* Get module state for struct sequence types */ PyObject *module = PyImport_ImportModule("_remote_debugging"); @@ -978,17 +1015,20 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre while (offset < reader->sample_data_size) { /* Read thread_id (8 bytes) + interpreter_id (4 bytes) + encoding byte */ if (reader->sample_data_size - offset < SAMPLE_HEADER_FIXED_SIZE) { - break; /* End of data */ + PyErr_Format(PyExc_ValueError, + "Truncated sample data: %zu trailing bytes", + reader->sample_data_size - offset); + return -1; } /* Use memcpy to avoid strict aliasing violations, then byte-swap if needed */ uint64_t thread_id_raw; uint32_t interpreter_id_raw; - memcpy(&thread_id_raw, &reader->sample_data[offset], sizeof(thread_id_raw)); - offset += 8; + memcpy(&thread_id_raw, &reader->sample_data[offset], SMP_SIZE_THREAD_ID); + offset += SMP_SIZE_THREAD_ID; - memcpy(&interpreter_id_raw, &reader->sample_data[offset], sizeof(interpreter_id_raw)); - offset += 4; + memcpy(&interpreter_id_raw, &reader->sample_data[offset], SMP_SIZE_INTERPRETER_ID); + offset += SMP_SIZE_INTERPRETER_ID; uint64_t thread_id = SWAP64_IF(reader->needs_swap, thread_id_raw); uint32_t interpreter_id = SWAP32_IF(reader->needs_swap, interpreter_id_raw); @@ -1024,6 +1064,11 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre count, max_possible_samples); return -1; } + if ((uint64_t)count > (uint64_t)PY_SSIZE_T_MAX - (uint64_t)replayed) { + PyErr_SetString(PyExc_OverflowError, + "Sample count exceeds Py_ssize_t maximum"); + return -1; + } reader->stats.repeat_records++; reader->stats.repeat_samples += count; @@ -1154,6 +1199,11 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre return -1; } Py_DECREF(timestamps_list); + if (replayed == PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "Sample count exceeds Py_ssize_t maximum"); + return -1; + } replayed++; reader->stats.total_samples++; break; @@ -1172,6 +1222,13 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre } } + if ((uint64_t)replayed != reader->sample_count) { + PyErr_Format(PyExc_ValueError, + "Sample count mismatch: header declares %u samples but replay decoded %zd", + reader->sample_count, replayed); + return -1; + } + /* Final progress callback at 100% */ if (invoke_progress_callback(progress_callback, replayed, reader->sample_count) < 0) { return -1; @@ -1259,8 +1316,6 @@ binary_reader_close(BinaryReader *reader) return; } - PyMem_Free(reader->filename); - #if USE_MMAP if (reader->mapped_data) { munmap(reader->mapped_data, reader->mapped_size); @@ -1270,13 +1325,9 @@ binary_reader_close(BinaryReader *reader) /* Clear sample_data which may point into the now-unmapped region */ reader->sample_data = NULL; reader->sample_data_size = 0; - if (reader->fd >= 0) { - close(reader->fd); - reader->fd = -1; /* Mark as closed */ - } #else if (reader->fp) { - fclose(reader->fp); + Py_fclose(reader->fp); reader->fp = NULL; } if (reader->file_data) { diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 0ac6c88d0373a77..341f9f7dc8ac457 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -29,9 +29,6 @@ /* Frame buffer: depth varint (max 2 bytes for 256) + 256 frames * 5 bytes/varint + margin */ #define MAX_FRAME_BUFFER_SIZE ((MAX_STACK_DEPTH * MAX_VARINT_SIZE_U32) + MAX_VARINT_SIZE_U32 + 16) -/* File structure sizes */ -#define FILE_FOOTER_SIZE 32 - /* Helper macro: convert PyLong to int32, using default_val if conversion fails */ #define PYLONG_TO_INT32_OR_DEFAULT(obj, var, default_val) \ do { \ @@ -111,7 +108,15 @@ fwrite_checked_allow_threads(const void *data, size_t size, FILE *fp) written = fwrite(data, 1, size, fp); Py_END_ALLOW_THREADS if (written != size) { - PyErr_SetFromErrno(PyExc_IOError); + int err = errno; + if (ferror(fp) && err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_IOError); + } + else { + PyErr_Format(PyExc_IOError, + "short write: wrote %zu of %zu bytes", written, size); + } return -1; } return 0; @@ -369,6 +374,11 @@ writer_intern_string(BinaryWriter *writer, PyObject *string, uint32_t *index) return 0; } + if (writer->string_count >= UINT32_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many strings for binary format"); + return -1; + } if (writer->string_count >= writer->string_capacity) { if (grow_parallel_arrays((void **)&writer->strings, (void **)&writer->string_lengths, @@ -383,6 +393,12 @@ writer_intern_string(BinaryWriter *writer, PyObject *string, uint32_t *index) if (!str_data) { return -1; } + if ((uintmax_t)str_len > UINT32_MAX) { + PyErr_Format(PyExc_OverflowError, + "string length %zd exceeds binary format maximum %u", + str_len, UINT32_MAX); + return -1; + } char *str_copy = PyMem_Malloc(str_len + 1); if (!str_copy) { @@ -425,6 +441,11 @@ writer_intern_frame(BinaryWriter *writer, const FrameEntry *entry, uint32_t *ind return 0; } + if (writer->frame_count >= UINT32_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many frames for binary format"); + return -1; + } if (GROW_ARRAY(writer->frame_entries, writer->frame_count, writer->frame_capacity, FrameEntry) < 0) { return -1; @@ -469,6 +490,11 @@ writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, } } + if (writer->thread_count >= UINT32_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many threads for binary format"); + return NULL; + } if (writer->thread_count >= writer->thread_capacity) { ThreadEntry *new_entries = grow_array(writer->thread_entries, &writer->thread_capacity, @@ -487,7 +513,7 @@ writer_get_or_create_thread_entry(BinaryWriter *writer, uint64_t thread_id, entry->prev_stack_capacity = MAX_STACK_DEPTH; entry->pending_rle_capacity = INITIAL_RLE_CAPACITY; - entry->prev_stack = PyMem_Malloc(entry->prev_stack_capacity * sizeof(uint32_t)); + entry->prev_stack = PyMem_Calloc(entry->prev_stack_capacity, sizeof(uint32_t)); if (!entry->prev_stack) { PyErr_NoMemory(); return NULL; @@ -588,9 +614,9 @@ static inline int write_sample_header(BinaryWriter *writer, ThreadEntry *entry, uint8_t encoding) { uint8_t header[SAMPLE_HEADER_FIXED_SIZE]; - memcpy(header, &entry->thread_id, 8); - memcpy(header + 8, &entry->interpreter_id, 4); - header[12] = encoding; + memcpy(header + SMP_OFF_THREAD_ID, &entry->thread_id, SMP_SIZE_THREAD_ID); + memcpy(header + SMP_OFF_INTERPRETER_ID, &entry->interpreter_id, SMP_SIZE_INTERPRETER_ID); + header[SMP_OFF_ENCODING] = encoding; return writer_write_bytes(writer, header, SAMPLE_HEADER_FIXED_SIZE); } @@ -603,6 +629,11 @@ flush_pending_rle(BinaryWriter *writer, ThreadEntry *entry) if (!entry->has_pending_rle || entry->pending_rle_count == 0) { return 0; } + if (entry->pending_rle_count > UINT32_MAX - writer->total_samples) { + PyErr_SetString(PyExc_OverflowError, + "too many samples for binary format"); + return -1; + } /* Write RLE record: * [thread_id: 8] [interpreter_id: 4] [STACK_REPEAT: 1] [count: varint] @@ -647,11 +678,17 @@ write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, const uint32_t *frame_indices, size_t stack_depth, size_t shared_count, size_t pop_count, size_t push_count) { + if (writer->total_samples == UINT32_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many samples for binary format"); + return -1; + } + /* Header: thread_id(8) + interpreter_id(4) + encoding(1) + delta(varint) + status(1) */ uint8_t header_buf[SAMPLE_HEADER_MAX_SIZE]; - memcpy(header_buf, &entry->thread_id, 8); - memcpy(header_buf + 8, &entry->interpreter_id, 4); - header_buf[12] = (uint8_t)encoding_type; + memcpy(header_buf + SMP_OFF_THREAD_ID, &entry->thread_id, SMP_SIZE_THREAD_ID); + memcpy(header_buf + SMP_OFF_INTERPRETER_ID, &entry->interpreter_id, SMP_SIZE_INTERPRETER_ID); + header_buf[SMP_OFF_ENCODING] = (uint8_t)encoding_type; size_t varint_len = encode_varint_u64( header_buf + SAMPLE_HEADER_FIXED_SIZE, timestamp_delta); @@ -720,7 +757,7 @@ write_sample_with_encoding(BinaryWriter *writer, ThreadEntry *entry, } BinaryWriter * -binary_writer_create(const char *filename, uint64_t sample_interval_us, int compression_type, +binary_writer_create(PyObject *path, uint64_t sample_interval_us, int compression_type, uint64_t start_time_us) { BinaryWriter *writer = PyMem_Calloc(1, sizeof(BinaryWriter)); @@ -729,14 +766,6 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp return NULL; } - writer->filename = PyMem_Malloc(strlen(filename) + 1); - if (!writer->filename) { - PyMem_Free(writer); - PyErr_NoMemory(); - return NULL; - } - strcpy(writer->filename, filename); - writer->start_time_us = start_time_us; writer->sample_interval_us = sample_interval_us; writer->compression_type = compression_type; @@ -802,9 +831,8 @@ binary_writer_create(const char *filename, uint64_t sample_interval_us, int comp } } - writer->fp = fopen(filename, "wb"); + writer->fp = Py_fopen(path, "wb"); if (!writer->fp) { - PyErr_SetFromErrnoWithFilename(PyExc_IOError, filename); goto error; } @@ -941,9 +969,8 @@ process_thread_sample(BinaryWriter *writer, PyObject *thread_info, } uint8_t status = (uint8_t)status_long; - int is_new_thread = 0; ThreadEntry *entry = writer_get_or_create_thread_entry( - writer, thread_id, interpreter_id, &is_new_thread); + writer, thread_id, interpreter_id, NULL); if (!entry) { return -1; } @@ -966,8 +993,15 @@ process_thread_sample(BinaryWriter *writer, PyObject *thread_info, curr_stack, curr_depth, &shared_count, &pop_count, &push_count); - if (encoding == STACK_REPEAT && !is_new_thread) { - /* Buffer this sample for RLE */ + if (encoding == STACK_REPEAT) { + /* Buffer this sample for RLE. + * + * STACK_REPEAT also covers the "first sample for a fresh thread, + * empty stack" case: a new ThreadEntry has prev_stack_depth == 0 + * and a zero-initialized prev_stack, so compare_stacks() returns + * STACK_REPEAT against an empty curr_stack (depth 0). Buffering + * it here is correct; the RLE flush path emits it as a normal + * STACK_REPEAT record. */ if (GROW_ARRAY(entry->pending_rle, entry->pending_rle_count, entry->pending_rle_capacity, PendingRLESample) < 0) { return -1; @@ -1145,17 +1179,17 @@ binary_writer_finalize(BinaryWriter *writer) PyErr_SetFromErrno(PyExc_IOError); return -1; } - uint64_t file_size = (uint64_t)footer_offset + 32; - uint8_t footer[32] = {0}; + uint64_t file_size = (uint64_t)footer_offset + FILE_FOOTER_SIZE; + uint8_t footer[FILE_FOOTER_SIZE] = {0}; /* Cast size_t to uint32_t before memcpy to ensure correct bytes are copied * on both little-endian and big-endian systems (size_t is 8 bytes on 64-bit) */ uint32_t string_count_u32 = (uint32_t)writer->string_count; uint32_t frame_count_u32 = (uint32_t)writer->frame_count; - memcpy(footer + 0, &string_count_u32, 4); - memcpy(footer + 4, &frame_count_u32, 4); - memcpy(footer + 8, &file_size, 8); - /* bytes 16-31: checksum placeholder (zeros) */ - if (fwrite_checked_allow_threads(footer, 32, writer->fp) < 0) { + memcpy(footer + FTR_OFF_STRINGS, &string_count_u32, FTR_SIZE_STRINGS); + memcpy(footer + FTR_OFF_FRAMES, &frame_count_u32, FTR_SIZE_FRAMES); + memcpy(footer + FTR_OFF_FILE_SIZE, &file_size, FTR_SIZE_FILE_SIZE); + /* checksum (FTR_OFF_CHECKSUM..FILE_FOOTER_SIZE-1): placeholder zeros */ + if (fwrite_checked_allow_threads(footer, FILE_FOOTER_SIZE, writer->fp) < 0) { return -1; } @@ -1190,7 +1224,7 @@ binary_writer_finalize(BinaryWriter *writer) return -1; } - if (fclose(writer->fp) != 0) { + if (Py_fclose(writer->fp) != 0) { writer->fp = NULL; PyErr_SetFromErrno(PyExc_IOError); return -1; @@ -1208,10 +1242,9 @@ binary_writer_destroy(BinaryWriter *writer) } if (writer->fp) { - fclose(writer->fp); + Py_fclose(writer->fp); } - PyMem_Free(writer->filename); PyMem_Free(writer->write_buffer); #ifdef HAVE_ZSTD diff --git a/Modules/_remote_debugging/clinic/module.c.h b/Modules/_remote_debugging/clinic/module.c.h index 179a7b97dd4e2f5..d01f3d13e85f09f 100644 --- a/Modules/_remote_debugging/clinic/module.c.h +++ b/Modules/_remote_debugging/clinic/module.c.h @@ -21,33 +21,37 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, "\n" "Args:\n" " pid: Process ID of the target Python process to debug\n" -" all_threads: If True, initialize state for all threads in the process.\n" -" If False, only initialize for the main thread.\n" +" all_threads: If True, initialize state for all threads in the\n" +" process. If False, only initialize for the main thread.\n" " only_active_thread: If True, only sample the thread holding the GIL.\n" -" mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL (gil-time).\n" -" Cannot be used together with all_threads=True.\n" -" debug: If True, chain exceptions to explain the sequence of events that\n" -" lead to the exception.\n" -" skip_non_matching_threads: If True, skip threads that don\'t match the selected mode.\n" -" If False, include all threads regardless of mode.\n" -" native: If True, include artificial \"<native>\" frames to denote calls to\n" -" non-Python code.\n" -" gc: If True, include artificial \"<GC>\" frames to denote active garbage\n" -" collection.\n" -" opcodes: If True, gather bytecode opcode information for instruction-level\n" -" profiling.\n" -" cache_frames: If True, enable frame caching optimization to avoid re-reading\n" -" unchanged parent frames between samples.\n" -" stats: If True, collect statistics about cache hits, memory reads, etc.\n" -" Use get_stats() to retrieve the collected statistics.\n" -"\n" -"The RemoteUnwinder provides functionality to inspect and debug a running Python\n" -"process, including examining thread states, stack frames and other runtime data.\n" +" mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL\n" +" (gil-time). Cannot be used together with all_threads=True.\n" +" debug: If True, chain exceptions to explain the sequence of events\n" +" that lead to the exception.\n" +" skip_non_matching_threads: If True, skip threads that don\'t match\n" +" the selected mode. If False, include all threads regardless of\n" +" mode.\n" +" native: If True, include artificial \"<native>\" frames to denote\n" +" calls to non-Python code.\n" +" gc: If True, include artificial \"<GC>\" frames to denote active\n" +" garbage collection.\n" +" opcodes: If True, gather bytecode opcode information for\n" +" instruction-level profiling.\n" +" cache_frames: If True, enable frame caching optimization to avoid\n" +" re-reading unchanged parent frames between samples.\n" +" stats: If True, collect statistics about cache hits, memory reads,\n" +" etc. Use get_stats() to retrieve the collected statistics.\n" +"\n" +"The RemoteUnwinder provides functionality to inspect and debug a running\n" +"Python process, including examining thread states, stack frames and\n" +"other runtime data.\n" "\n" "Raises:\n" " PermissionError: If access to the target process is denied\n" -" OSError: If unable to attach to the target process or access its memory\n" -" RuntimeError: If unable to read debug information from the target process\n" +" OSError: If unable to attach to the target process or access its\n" +" memory\n" +" RuntimeError: If unable to read debug information from the target\n" +" process\n" " ValueError: If both all_threads and only_active_thread are True"); static int @@ -217,16 +221,21 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, "\n" "Returns stack traces for all interpreters and threads in process.\n" "\n" -"Each element in the returned list is a tuple of (interpreter_id, thread_list), where:\n" +"Each element in the returned list is a tuple of (interpreter_id,\n" +"thread_list), where:\n" "- interpreter_id is the interpreter identifier\n" -"- thread_list is a list of tuples (thread_id, frame_list) for threads in that interpreter\n" +"- thread_list is a list of tuples (thread_id, frame_list) for\n" +" threads in that interpreter\n" " - thread_id is the OS thread identifier\n" -" - frame_list is a list of tuples (function_name, filename, line_number) representing\n" -" the Python stack frames for that thread, ordered from most recent to oldest\n" +" - frame_list is a list of tuples (function_name, filename,\n" +" line_number) representing the Python stack frames for that\n" +" thread, ordered from most recent to oldest\n" "\n" "The threads returned depend on the initialization parameters:\n" -"- If only_active_thread was True: returns only the thread holding the GIL across all interpreters\n" -"- If all_threads was True: returns all threads across all interpreters\n" +"- If only_active_thread was True: returns only the thread holding\n" +" the GIL across all interpreters\n" +"- If all_threads was True: returns all threads across all\n" +" interpreters\n" "- Otherwise: returns only the main thread of each interpreter\n" "\n" "Example:\n" @@ -250,10 +259,12 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, " ]\n" "\n" "Raises:\n" -" RuntimeError: If there is an error copying memory from the target process\n" +" RuntimeError: If there is an error copying memory from the\n" +" target process\n" " OSError: If there is an error accessing the target process\n" " PermissionError: If access to the target process is denied\n" -" UnicodeDecodeError: If there is an error decoding strings from the target process"); +" UnicodeDecodeError: If there is an error decoding strings from\n" +" the target process"); #define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF \ {"get_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_stack_trace__doc__}, @@ -279,20 +290,25 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, "\n" "Get all tasks and their awaited_by relationships from the remote process.\n" "\n" -"This provides a tree structure showing which tasks are waiting for other tasks.\n" +"This provides a tree structure showing which tasks are waiting for\n" +"other tasks.\n" "\n" "For each task, returns:\n" -"1. The call stack frames leading to where the task is currently executing\n" +"1. The call stack frames leading to where the task is currently\n" +" executing\n" "2. The name of the task\n" -"3. A list of tasks that this task is waiting for, with their own frames/names/etc\n" +"3. A list of tasks that this task is waiting for, with their own\n" +" frames/names/etc\n" "\n" "Returns a list of [frames, task_name, subtasks] where:\n" -"- frames: List of (func_name, filename, lineno) showing the call stack\n" +"- frames: List of (func_name, filename, lineno) showing the call\n" +" stack\n" "- task_name: String identifier for the task\n" "- subtasks: List of tasks being awaited by this task, in same format\n" "\n" "Raises:\n" -" RuntimeError: If AsyncioDebug section is not available in the remote process\n" +" RuntimeError: If AsyncioDebug section is not available in the\n" +" remote process\n" " MemoryError: If memory allocation fails\n" " OSError: If reading from the remote process fails\n" "\n" @@ -336,14 +352,16 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, "\n" "Get the currently running async tasks and their dependency graphs from the remote process.\n" "\n" -"This returns information about running tasks and all tasks that are waiting for them,\n" -"forming a complete dependency graph for each thread\'s active task.\n" +"This returns information about running tasks and all tasks that are\n" +"waiting for them, forming a complete dependency graph for each\n" +"thread\'s active task.\n" "\n" -"For each thread with a running task, returns the running task plus all tasks that\n" -"transitively depend on it (tasks waiting for the running task, tasks waiting for\n" -"those tasks, etc.).\n" +"For each thread with a running task, returns the running task plus\n" +"all tasks that transitively depend on it (tasks waiting for the\n" +"running task, tasks waiting for those tasks, etc.).\n" "\n" -"Returns a list of per-thread results, where each thread result contains:\n" +"Returns a list of per-thread results, where each thread result\n" +"contains:\n" "- Thread ID\n" "- List of task information for the running task and all its waiters\n" "\n" @@ -354,11 +372,13 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, "- List of tasks waiting for this task (recursive structure)\n" "\n" "Raises:\n" -" RuntimeError: If AsyncioDebug section is not available in the target process\n" +" RuntimeError: If AsyncioDebug section is not available in the\n" +" target process\n" " MemoryError: If memory allocation fails\n" " OSError: If reading from the remote process fails\n" "\n" -"Example output (similar structure to get_all_awaited_by but only for running tasks):\n" +"Example output (similar structure to get_all_awaited_by but only for\n" +"running tasks):\n" "[\n" " (140234, [\n" " (4345585712, \'main_task\',\n" @@ -403,16 +423,34 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stats__doc__, " - total_samples: Total number of get_stack_trace calls\n" " - frame_cache_hits: Full cache hits (entire stack unchanged)\n" " - frame_cache_misses: Cache misses requiring full walk\n" -" - frame_cache_partial_hits: Partial hits (stopped at cached frame)\n" +" - frame_cache_partial_hits: Partial hits (stopped at cached\n" +" frame)\n" " - frames_read_from_cache: Total frames retrieved from cache\n" -" - frames_read_from_memory: Total frames read from remote memory\n" +" - frames_read_from_memory: Total frames read from remote\n" +" memory\n" " - memory_reads: Total remote memory read operations\n" " - memory_bytes_read: Total bytes read from remote memory\n" " - code_object_cache_hits: Code object cache hits\n" " - code_object_cache_misses: Code object cache misses\n" -" - stale_cache_invalidations: Times stale cache entries were cleared\n" -" - frame_cache_hit_rate: Percentage of samples that hit the cache\n" -" - code_object_cache_hit_rate: Percentage of code object lookups that hit cache\n" +" - stale_cache_invalidations: Times stale cache entries were\n" +" cleared\n" +" - batched_read_attempts: Batched remote-read attempts\n" +" - batched_read_successes: Attempts that read all requested\n" +" segments\n" +" - batched_read_misses: Attempts that fell back or partially\n" +" read\n" +" - batched_read_segments_requested: Segments requested by\n" +" batched reads\n" +" - batched_read_segments_completed: Segments completed by\n" +" batched reads\n" +" - frame_cache_hit_rate: Percentage of samples that hit the\n" +" cache\n" +" - code_object_cache_hit_rate: Percentage of code object\n" +" lookups that hit cache\n" +" - batched_read_success_rate: Percentage of batched reads\n" +" that completed all segments\n" +" - batched_read_segment_completion_rate: Percentage of\n" +" requested segments read by batched reads\n" "\n" "Raises:\n" " RuntimeError: If stats collection was not enabled (stats=False)"); @@ -442,9 +480,11 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_pause_threads__doc__, "Pause all threads in the target process.\n" "\n" "This stops all threads in the target process to allow for consistent\n" -"memory reads during sampling. Must be paired with a call to resume_threads().\n" +"memory reads during sampling. Must be paired with a call to\n" +"resume_threads().\n" "\n" -"Returns True if threads were successfully paused, False if they were already paused.\n" +"Returns True if threads were successfully paused, False if they were\n" +"already paused.\n" "\n" "Raises:\n" " RuntimeError: If there is an error stopping the threads"); @@ -473,9 +513,11 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_resume_threads__doc__, "\n" "Resume all threads in the target process.\n" "\n" -"This resumes threads that were previously paused with pause_threads().\n" +"This resumes threads that were previously paused with\n" +"pause_threads().\n" "\n" -"Returns True if threads were successfully resumed, False if they were not paused."); +"Returns True if threads were successfully resumed, False if they\n" +"were not paused."); #define _REMOTE_DEBUGGING_REMOTEUNWINDER_RESUME_THREADS_METHODDEF \ {"resume_threads", (PyCFunction)_remote_debugging_RemoteUnwinder_resume_threads, METH_NOARGS, _remote_debugging_RemoteUnwinder_resume_threads__doc__}, @@ -503,16 +545,18 @@ PyDoc_STRVAR(_remote_debugging_GCMonitor___init____doc__, "\n" "Args:\n" " pid: Process ID of the target Python process to monitor\n" -" debug: If True, chain exceptions to explain the sequence of events that\n" -" lead to the exception.\n" +" debug: If True, chain exceptions to explain the sequence of\n" +" events that lead to the exception.\n" "\n" -"The GCMonitor provides functionality to read GC statistics from a running\n" -"Python process.\n" +"The GCMonitor provides functionality to read GC statistics from\n" +"a running Python process.\n" "\n" "Raises:\n" " PermissionError: If access to the target process is denied\n" -" OSError: If unable to attach to the target process or access its memory\n" -" RuntimeError: If unable to read debug information from the target process"); +" OSError: If unable to attach to the target process or access\n" +" its memory\n" +" RuntimeError: If unable to read debug information from the\n" +" target process"); static int _remote_debugging_GCMonitor___init___impl(GCMonitorObject *self, int pid, @@ -601,11 +645,12 @@ PyDoc_STRVAR(_remote_debugging_GCMonitor_get_gc_stats__doc__, " - collected: Total number of collected objects.\n" " - uncollectable: Total number of uncollectable objects.\n" " - candidates: Total objects considered and traversed.\n" +" - heap_size: number of live objects.\n" " - duration: Total collection time, in seconds.\n" "\n" "Raises:\n" -" RuntimeError: If the target process cannot be inspected or if its\n" -" debug offsets or GC stats layout are incompatible."); +" RuntimeError: If the target process cannot be inspected or if\n" +" its debug offsets or GC stats layout are incompatible."); #define _REMOTE_DEBUGGING_GCMONITOR_GET_GC_STATS_METHODDEF \ {"get_gc_stats", _PyCFunction_CAST(_remote_debugging_GCMonitor_get_gc_stats), METH_FASTCALL|METH_KEYWORDS, _remote_debugging_GCMonitor_get_gc_stats__doc__}, @@ -680,14 +725,15 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter___init____doc__, "Arguments:\n" " filename: Path to output file\n" " sample_interval_us: Sampling interval in microseconds\n" -" start_time_us: Start timestamp in microseconds (from time.monotonic() * 1e6)\n" +" start_time_us: Start timestamp in microseconds (from\n" +" time.monotonic() * 1e6)\n" " compression: 0=none, 1=zstd (default: 0)\n" "\n" "Use as a context manager or call finalize() when done."); static int _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, - const char *filename, + PyObject *filename, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression); @@ -727,7 +773,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args, PyObject PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 3; - const char *filename; + PyObject *filename; unsigned long long sample_interval_us; unsigned long long start_time_us; int compression = 0; @@ -737,19 +783,7 @@ _remote_debugging_BinaryWriter___init__(PyObject *self, PyObject *args, PyObject if (!fastargs) { goto exit; } - if (!PyUnicode_Check(fastargs[0])) { - _PyArg_BadArgument("BinaryWriter", "argument 'filename'", "str", fastargs[0]); - goto exit; - } - Py_ssize_t filename_length; - filename = PyUnicode_AsUTF8AndSize(fastargs[0], &filename_length); - if (filename == NULL) { - goto exit; - } - if (strlen(filename) != (size_t)filename_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + filename = fastargs[0]; if (!_PyLong_UnsignedLongLong_Converter(fastargs[1], &sample_interval_us)) { goto exit; } @@ -778,7 +812,8 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter_write_sample__doc__, "\n" "Arguments:\n" " stack_frames: List of InterpreterInfo objects\n" -" timestamp_us: Current timestamp in microseconds (from time.monotonic() * 1e6)"); +" timestamp_us: Current timestamp in microseconds (from\n" +" time.monotonic() * 1e6)"); #define _REMOTE_DEBUGGING_BINARYWRITER_WRITE_SAMPLE_METHODDEF \ {"write_sample", _PyCFunction_CAST(_remote_debugging_BinaryWriter_write_sample), METH_FASTCALL|METH_KEYWORDS, _remote_debugging_BinaryWriter_write_sample__doc__}, @@ -980,8 +1015,9 @@ PyDoc_STRVAR(_remote_debugging_BinaryWriter_get_stats__doc__, "\n" "Get encoding statistics for the writer.\n" "\n" -"Returns a dict with encoding statistics including repeat/full/suffix/pop-push\n" -"record counts, frames written/saved, and compression ratio."); +"Returns a dict with encoding statistics including\n" +"repeat/full/suffix/pop-push record counts, frames written/saved, and\n" +"compression ratio."); #define _REMOTE_DEBUGGING_BINARYWRITER_GET_STATS_METHODDEF \ {"get_stats", (PyCFunction)_remote_debugging_BinaryWriter_get_stats, METH_NOARGS, _remote_debugging_BinaryWriter_get_stats__doc__}, @@ -1008,7 +1044,7 @@ PyDoc_STRVAR(_remote_debugging_BinaryReader___init____doc__, static int _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, - const char *filename); + PyObject *filename); static int _remote_debugging_BinaryReader___init__(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1044,26 +1080,14 @@ _remote_debugging_BinaryReader___init__(PyObject *self, PyObject *args, PyObject PyObject *argsbuf[1]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - const char *filename; + PyObject *filename; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (!PyUnicode_Check(fastargs[0])) { - _PyArg_BadArgument("BinaryReader", "argument 'filename'", "str", fastargs[0]); - goto exit; - } - Py_ssize_t filename_length; - filename = PyUnicode_AsUTF8AndSize(fastargs[0], &filename_length); - if (filename == NULL) { - goto exit; - } - if (strlen(filename) != (size_t)filename_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + filename = fastargs[0]; return_value = _remote_debugging_BinaryReader___init___impl((BinaryReaderObject *)self, filename); exit: @@ -1171,8 +1195,8 @@ PyDoc_STRVAR(_remote_debugging_BinaryReader_get_stats__doc__, "\n" "Get reconstruction statistics from replay.\n" "\n" -"Returns a dict with statistics about record types decoded and samples\n" -"reconstructed during replay."); +"Returns a dict with statistics about record types decoded and\n" +"samples reconstructed during replay."); #define _REMOTE_DEBUGGING_BINARYREADER_GET_STATS_METHODDEF \ {"get_stats", (PyCFunction)_remote_debugging_BinaryReader_get_stats, METH_NOARGS, _remote_debugging_BinaryReader_get_stats__doc__}, @@ -1335,11 +1359,12 @@ PyDoc_STRVAR(_remote_debugging_get_child_pids__doc__, " If True, return all descendants (children, grandchildren, etc.).\n" " If False, return only direct children.\n" "\n" -"Returns a list of child process IDs. Returns an empty list if no children\n" -"are found.\n" +"Returns a list of child process IDs. Returns an empty list if no\n" +"children are found.\n" "\n" -"This function provides a snapshot of child processes at a moment in time.\n" -"Child processes may exit or new ones may be created after the list is returned.\n" +"This function provides a snapshot of child processes at a moment in\n" +"time. Child processes may exit or new ones may be created after the\n" +"list is returned.\n" "\n" "Raises:\n" " OSError: If unable to enumerate processes\n" @@ -1563,4 +1588,4 @@ _remote_debugging_get_gc_stats(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=1151e58683dab9f4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a3df14a6ab7f2998 input=a9049054013a1b77]*/ diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index 7b95c0f2d4fa8da..ab889a130ee4e7e 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -47,7 +47,6 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t // Read the TLBC array pointer if (read_ptr(unwinder, tlbc_array_addr, &tlbc_array_ptr) != 0) { - PyErr_SetString(PyExc_RuntimeError, "Failed to read TLBC array pointer"); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array pointer"); return 0; // Read error } @@ -61,7 +60,6 @@ cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t // Read the TLBC array size Py_ssize_t tlbc_size; if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, tlbc_array_ptr, sizeof(tlbc_size), &tlbc_size) != 0) { - PyErr_SetString(PyExc_RuntimeError, "Failed to read TLBC array size"); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array size"); return 0; // Read error } @@ -405,6 +403,8 @@ parse_code_object(RemoteUnwinderObject *unwinder, meta->func_name = func; meta->file_name = file; meta->linetable = linetable; + meta->last_frame_info = NULL; + meta->last_addrq = -1; meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; @@ -432,7 +432,7 @@ parse_code_object(RemoteUnwinderObject *unwinder, #ifdef Py_GIL_DISABLED // Handle thread-local bytecode (TLBC) in free threading builds - if (ctx->tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) { + if (ctx->tlbc_index == 0 || unwinder == NULL || unwinder->debug_offsets.code_object.co_tlbc == 0) { // No TLBC or no unwinder - use main bytecode directly addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; goto done_tlbc; @@ -482,6 +482,12 @@ parse_code_object(RemoteUnwinderObject *unwinder, addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; #endif ; // Empty statement to avoid C23 extension warning + + if (!unwinder->opcodes && meta->last_frame_info != NULL && meta->last_addrq == addrq) { + *result = Py_NewRef(meta->last_frame_info); + return 0; + } + LocationInfo info = {0}; bool ok = parse_linetable(addrq, PyBytes_AS_STRING(meta->linetable), PyBytes_GET_SIZE(meta->linetable), @@ -529,6 +535,11 @@ parse_code_object(RemoteUnwinderObject *unwinder, goto error; } + if (!unwinder->opcodes) { + Py_XSETREF(meta->last_frame_info, Py_NewRef(tuple)); + meta->last_addrq = addrq; + } + *result = tuple; return 0; diff --git a/Modules/_remote_debugging/debug_offsets_validation.h b/Modules/_remote_debugging/debug_offsets_validation.h index 1507026306192e3..f070f03ac459dcd 100644 --- a/Modules/_remote_debugging/debug_offsets_validation.h +++ b/Modules/_remote_debugging/debug_offsets_validation.h @@ -31,7 +31,7 @@ #define FIELD_SIZE(type, member) sizeof(((type *)0)->member) enum { - PY_REMOTE_DEBUG_OFFSETS_TOTAL_SIZE = 848, + PY_REMOTE_DEBUG_OFFSETS_TOTAL_SIZE = 880, PY_REMOTE_ASYNC_DEBUG_OFFSETS_TOTAL_SIZE = 104, }; diff --git a/Modules/_remote_debugging/frame_cache.c b/Modules/_remote_debugging/frame_cache.c index b6566d7cff7b543..19fc406bca9ac96 100644 --- a/Modules/_remote_debugging/frame_cache.c +++ b/Modules/_remote_debugging/frame_cache.c @@ -30,6 +30,7 @@ frame_cache_cleanup(RemoteUnwinderObject *unwinder) return; } for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + Py_CLEAR(unwinder->frame_cache[i].thread_id_obj); Py_CLEAR(unwinder->frame_cache[i].frame_list); } PyMem_Free(unwinder->frame_cache); @@ -53,6 +54,21 @@ frame_cache_find(RemoteUnwinderObject *unwinder, uint64_t thread_id) return NULL; } +FrameCacheEntry * +frame_cache_find_by_tstate(RemoteUnwinderObject *unwinder, uintptr_t tstate_addr) +{ + if (!unwinder->frame_cache || tstate_addr == 0) { + return NULL; + } + for (int i = 0; i < FRAME_CACHE_MAX_THREADS; i++) { + if (unwinder->frame_cache[i].thread_state_addr == tstate_addr) { + assert(unwinder->frame_cache[i].num_addrs <= FRAME_CACHE_MAX_FRAMES); + return &unwinder->frame_cache[i]; + } + } + return NULL; +} + // Allocate a cache slot for a thread // Returns NULL if cache is full (graceful degradation) static FrameCacheEntry * @@ -127,8 +143,10 @@ frame_cache_invalidate_stale(RemoteUnwinderObject *unwinder, PyObject *result) } if (!found) { // Clear this entry + Py_CLEAR(unwinder->frame_cache[i].thread_id_obj); Py_CLEAR(unwinder->frame_cache[i].frame_list); unwinder->frame_cache[i].thread_id = 0; + unwinder->frame_cache[i].thread_state_addr = 0; unwinder->frame_cache[i].num_addrs = 0; STATS_INC(unwinder, stale_cache_invalidations); } @@ -216,6 +234,7 @@ frame_cache_store( PyObject *frame_list, const uintptr_t *addrs, Py_ssize_t num_addrs, + uintptr_t thread_state_addr, uintptr_t base_frame_addr, uintptr_t last_frame_visited) { @@ -257,6 +276,13 @@ frame_cache_store( return -1; } entry->thread_id = thread_id; + entry->thread_state_addr = thread_state_addr; + if (entry->thread_id_obj == NULL) { + entry->thread_id_obj = PyLong_FromUnsignedLongLong(thread_id); + if (entry->thread_id_obj == NULL) { + return -1; + } + } memcpy(entry->addrs, addrs, num_addrs * sizeof(uintptr_t)); entry->num_addrs = num_addrs; assert(entry->num_addrs == num_addrs); diff --git a/Modules/_remote_debugging/frames.c b/Modules/_remote_debugging/frames.c index bbdfce3f7201d9d..e7d2a2764390261 100644 --- a/Modules/_remote_debugging/frames.c +++ b/Modules/_remote_debugging/frames.c @@ -56,12 +56,14 @@ process_single_stack_chunk( return -1; } - this_chunk = PyMem_RawRealloc(this_chunk, actual_size); - if (!this_chunk) { + char *tmp = PyMem_RawRealloc(this_chunk, actual_size); + if (!tmp) { + PyMem_RawFree(this_chunk); PyErr_NoMemory(); set_exception_cause(unwinder, PyExc_MemoryError, "Failed to reallocate stack chunk buffer"); return -1; } + this_chunk = tmp; if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, chunk_addr, actual_size, this_chunk) < 0) { PyMem_RawFree(this_chunk); @@ -186,30 +188,16 @@ is_frame_valid( return 1; } -int -parse_frame_object( +static int +parse_frame_buffer( RemoteUnwinderObject *unwinder, PyObject** result, - uintptr_t address, + const char *frame, uintptr_t* address_of_code_object, uintptr_t* previous_frame ) { - char frame[SIZEOF_INTERP_FRAME]; *address_of_code_object = 0; - Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address, - SIZEOF_INTERP_FRAME, - frame - ); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); - return -1; - } - STATS_INC(unwinder, memory_reads); - STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); - *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); @@ -237,6 +225,31 @@ parse_frame_object( return parse_code_object(unwinder, result, &code_ctx); } +int +parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* address_of_code_object, + uintptr_t* previous_frame +) { + char frame[SIZEOF_INTERP_FRAME]; + Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_INTERP_FRAME, + frame + ); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); + return -1; + } + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + + return parse_frame_buffer(unwinder, result, frame, address_of_code_object, previous_frame); +} + int parse_frame_from_chunks( RemoteUnwinderObject *unwinder, @@ -312,15 +325,32 @@ process_frame_chain( } assert(frame_count <= MAX_FRAMES); - if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks) < 0) { + if (ctx->chunks && ctx->chunks->count > 0) { + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, &stackpointer, ctx->chunks) == 0) { + goto parsed_frame; + } PyErr_Clear(); + } + { uintptr_t address_of_code_object = 0; - if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object, &next_frame_addr) < 0) { + int parse_result; + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == frame_addr) { + parse_result = parse_frame_buffer( + unwinder, &frame, ctx->prefetch.frame, + &address_of_code_object, &next_frame_addr); + } + else { + parse_result = parse_frame_object( + unwinder, &frame, frame_addr, + &address_of_code_object, &next_frame_addr); + } + if (parse_result < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); return -1; } } +parsed_frame: // Skip first frame if requested (used for cache miss continuation) if (ctx->skip_first_frame && frame_count == 1) { Py_XDECREF(frame); @@ -501,41 +531,37 @@ try_full_cache_hit( PyObject *current_frame = NULL; uintptr_t code_object_addr = 0; uintptr_t previous_frame = 0; - int parse_result = parse_frame_object(unwinder, &current_frame, ctx->frame_addr, + int parse_result; + if (ctx->prefetch.frame && ctx->prefetch.frame_addr == ctx->frame_addr) { + parse_result = parse_frame_buffer(unwinder, &current_frame, + ctx->prefetch.frame, + &code_object_addr, &previous_frame); + } + else { + parse_result = parse_frame_object(unwinder, &current_frame, ctx->frame_addr, &code_object_addr, &previous_frame); + } if (parse_result < 0) { return -1; } - Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); - PyObject *parent_slice = NULL; - if (cached_size > 1) { - parent_slice = PyList_GetSlice(entry->frame_list, 1, cached_size); - if (!parent_slice) { - Py_XDECREF(current_frame); - return -1; - } - } - if (current_frame != NULL) { if (PyList_Append(ctx->frame_info, current_frame) < 0) { Py_DECREF(current_frame); - Py_XDECREF(parent_slice); return -1; } Py_DECREF(current_frame); STATS_ADD(unwinder, frames_read_from_memory, 1); } - if (parent_slice) { - Py_ssize_t cur_size = PyList_GET_SIZE(ctx->frame_info); - int result = PyList_SetSlice(ctx->frame_info, cur_size, cur_size, parent_slice); - Py_DECREF(parent_slice); - if (result < 0) { + Py_ssize_t cached_size = PyList_GET_SIZE(entry->frame_list); + for (Py_ssize_t i = 1; i < cached_size; i++) { + PyObject *cached_frame = PyList_GET_ITEM(entry->frame_list, i); + if (PyList_Append(ctx->frame_info, cached_frame) < 0) { return -1; } - STATS_ADD(unwinder, frames_read_from_cache, cached_size - 1); } + STATS_ADD(unwinder, frames_read_from_cache, cached_size > 1 ? cached_size - 1 : 0); STATS_INC(unwinder, frame_cache_hits); return 1; @@ -554,6 +580,14 @@ collect_frames_with_cache( return full_hit < 0 ? -1 : 0; } + assert(ctx->chunks != NULL); + + if (ctx->chunks->count == 0) { + if (copy_stack_chunks(unwinder, ctx->thread_state_addr, ctx->chunks) < 0) { + PyErr_Clear(); + } + } + Py_ssize_t frames_before = PyList_GET_SIZE(ctx->frame_info); if (process_frame_chain(unwinder, ctx) < 0) { @@ -606,7 +640,8 @@ collect_frames_with_cache( } if (frame_cache_store(unwinder, thread_id, ctx->frame_info, ctx->frame_addrs, ctx->num_addrs, - ctx->base_frame_addr, ctx->last_frame_visited) < 0) { + ctx->thread_state_addr, ctx->base_frame_addr, + ctx->last_frame_visited) < 0) { return -1; } diff --git a/Modules/_remote_debugging/gc_stats.c b/Modules/_remote_debugging/gc_stats.c index 852dc866153192c..d5d05edb8ecf5ee 100644 --- a/Modules/_remote_debugging/gc_stats.c +++ b/Modules/_remote_debugging/gc_stats.c @@ -53,6 +53,7 @@ read_gc_stats(struct gc_stats *stats, int64_t iid, PyObject *result, SET_FIELD(PyLong_FromSsize_t, items->collected); SET_FIELD(PyLong_FromSsize_t, items->uncollectable); SET_FIELD(PyLong_FromSsize_t, items->candidates); + SET_FIELD(PyLong_FromSsize_t, items->heap_size); SET_FIELD(PyFloat_FromDouble, items->duration); diff --git a/Modules/_remote_debugging/module.c b/Modules/_remote_debugging/module.c index c840c59971c4786..36115f20d9d4ccc 100644 --- a/Modules/_remote_debugging/module.c +++ b/Modules/_remote_debugging/module.c @@ -143,6 +143,7 @@ static PyStructSequence_Field GCStatsInfo_fields[] = { {"collected", "Total number of collected objects"}, {"uncollectable", "Total number of uncollectable objects"}, {"candidates", "Total objects considered and traversed"}, + {"heap_size", "Number of live objects"}, {"duration", "Total collection time, in seconds"}, {NULL} }; @@ -151,7 +152,7 @@ PyStructSequence_Desc GCStatsInfo_desc = { "_remote_debugging.GCStatsInfo", "Information about a garbage collector stats sample", GCStatsInfo_fields, - 9 + 10 }; /* ============================================================================ @@ -165,6 +166,7 @@ cached_code_metadata_destroy(void *ptr) Py_DECREF(meta->func_name); Py_DECREF(meta->file_name); Py_DECREF(meta->linetable); + Py_XDECREF(meta->last_frame_info); PyMem_RawFree(meta); } @@ -278,7 +280,6 @@ class _remote_debugging.RemoteUnwinder "RemoteUnwinderObject *" "&RemoteUnwinder /*[clinic input] @permit_long_summary -@permit_long_docstring_body _remote_debugging.RemoteUnwinder.__init__ pid: int * @@ -297,33 +298,37 @@ Initialize a new RemoteUnwinder object for debugging a remote Python process. Args: pid: Process ID of the target Python process to debug - all_threads: If True, initialize state for all threads in the process. - If False, only initialize for the main thread. + all_threads: If True, initialize state for all threads in the + process. If False, only initialize for the main thread. only_active_thread: If True, only sample the thread holding the GIL. - mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL (gil-time). - Cannot be used together with all_threads=True. - debug: If True, chain exceptions to explain the sequence of events that - lead to the exception. - skip_non_matching_threads: If True, skip threads that don't match the selected mode. - If False, include all threads regardless of mode. - native: If True, include artificial "<native>" frames to denote calls to - non-Python code. - gc: If True, include artificial "<GC>" frames to denote active garbage - collection. - opcodes: If True, gather bytecode opcode information for instruction-level - profiling. - cache_frames: If True, enable frame caching optimization to avoid re-reading - unchanged parent frames between samples. - stats: If True, collect statistics about cache hits, memory reads, etc. - Use get_stats() to retrieve the collected statistics. - -The RemoteUnwinder provides functionality to inspect and debug a running Python -process, including examining thread states, stack frames and other runtime data. + mode: Profiling mode: 0=WALL (wall-time), 1=CPU (cpu-time), 2=GIL + (gil-time). Cannot be used together with all_threads=True. + debug: If True, chain exceptions to explain the sequence of events + that lead to the exception. + skip_non_matching_threads: If True, skip threads that don't match + the selected mode. If False, include all threads regardless of + mode. + native: If True, include artificial "<native>" frames to denote + calls to non-Python code. + gc: If True, include artificial "<GC>" frames to denote active + garbage collection. + opcodes: If True, gather bytecode opcode information for + instruction-level profiling. + cache_frames: If True, enable frame caching optimization to avoid + re-reading unchanged parent frames between samples. + stats: If True, collect statistics about cache hits, memory reads, + etc. Use get_stats() to retrieve the collected statistics. + +The RemoteUnwinder provides functionality to inspect and debug a running +Python process, including examining thread states, stack frames and +other runtime data. Raises: PermissionError: If access to the target process is denied - OSError: If unable to attach to the target process or access its memory - RuntimeError: If unable to read debug information from the target process + OSError: If unable to attach to the target process or access its + memory + RuntimeError: If unable to read debug information from the target + process ValueError: If both all_threads and only_active_thread are True [clinic start generated code]*/ @@ -336,7 +341,7 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, int native, int gc, int opcodes, int cache_frames, int stats) -/*[clinic end generated code: output=0031f743f4b9ad52 input=8fb61b24102dec6e]*/ +/*[clinic end generated code: output=0031f743f4b9ad52 input=9d25ae328d62626d]*/ { // Validate that all_threads and only_active_thread are not both True if (all_threads && only_active_thread) { @@ -359,6 +364,10 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, self->cache_frames = cache_frames; self->collect_stats = stats; self->stale_invalidation_counter = 0; + self->cached_tstate_interpreter_addr = 0; + self->cached_tstate_addr = 0; + memset(self->cached_tstates, 0, sizeof(self->cached_tstates)); + memset(self->cached_generations, 0, sizeof(self->cached_generations)); self->debug = debug; self->only_active_thread = only_active_thread; self->mode = mode; @@ -402,6 +411,9 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return -1; } if (async_debug_result < 0) { + if (_Py_RemoteDebug_HasPermissionError()) { + return -1; + } PyErr_Clear(); memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets)); self->async_debug_offsets_available = 0; @@ -472,23 +484,193 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, return 0; } +static inline size_t +interpreter_thread_cache_index(uintptr_t interpreter_addr) +{ + // Direct-mapped table indexed by the remote interpreter address. Each entry + // stores the full address and verifies it on lookup, so hash collisions + // degrade to misses and cannot return a value from the wrong interpreter. + return (size_t)_Py_HashPointerRaw((const void *)interpreter_addr) + & (INTERPRETER_THREAD_CACHE_SIZE - 1); +} + +static inline uintptr_t +get_cached_tstate_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr) +{ + if (interpreter_addr == 0) { + return 0; + } + + if (self->cached_tstate_interpreter_addr == interpreter_addr) { + return self->cached_tstate_addr; + } + + InterpreterTstateCacheEntry *entry = + &self->cached_tstates[interpreter_thread_cache_index(interpreter_addr)]; + if (entry->interpreter_addr == interpreter_addr) { + self->cached_tstate_interpreter_addr = interpreter_addr; + self->cached_tstate_addr = entry->thread_state_addr; + return entry->thread_state_addr; + } + return 0; +} + +static inline void +set_cached_tstate_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr, + uintptr_t thread_state_addr) +{ + if (interpreter_addr == 0 || thread_state_addr == 0) { + return; + } + + self->cached_tstate_interpreter_addr = interpreter_addr; + self->cached_tstate_addr = thread_state_addr; + + InterpreterTstateCacheEntry *entry = + &self->cached_tstates[interpreter_thread_cache_index(interpreter_addr)]; + entry->interpreter_addr = interpreter_addr; + entry->thread_state_addr = thread_state_addr; +} + +static void +refresh_generation_caches_from_interp_state( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr, + const char *interp_state_buffer) +{ + uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, + self->debug_offsets.interpreter_state.code_object_generation); + + if (self->cached_generation_interpreter_addr == interpreter_addr) { + if (code_object_generation != self->cached_code_object_generation) { + self->cached_code_object_generation = code_object_generation; + _Py_hashtable_clear(self->code_object_cache); + } + } + else { + InterpreterGenerationCacheEntry *entry = + &self->cached_generations[interpreter_thread_cache_index(interpreter_addr)]; + // A slot rebound from another interpreter must be treated as changed: + // the code_object_cache is global, so even if the new generation + // numerically matches what the previous occupant had, stale entries + // from that occupant could still be served. + int changed = entry->interpreter_addr != interpreter_addr + || entry->code_object_generation != code_object_generation; + entry->interpreter_addr = interpreter_addr; + entry->code_object_generation = code_object_generation; + if (changed) { + _Py_hashtable_clear(self->code_object_cache); + } + self->cached_generation_interpreter_addr = interpreter_addr; + self->cached_code_object_generation = code_object_generation; + } + +#ifdef Py_GIL_DISABLED + uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, + self->debug_offsets.interpreter_state.tlbc_generation); + if (current_tlbc_generation != self->tlbc_generation) { + self->tlbc_generation = current_tlbc_generation; + _Py_hashtable_clear(self->tlbc_cache); + } +#endif +} + +static int +refresh_generation_caches_for_interpreter( + RemoteUnwinderObject *self, + uintptr_t interpreter_addr) +{ + char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; + if (_Py_RemoteDebug_ReadRemoteMemory( + &self->handle, + interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer) < 0) { + set_exception_cause(self, PyExc_RuntimeError, + "Failed to read interpreter state buffer"); + return -1; + } + refresh_generation_caches_from_interp_state(self, interpreter_addr, interp_state_buffer); + return 0; +} + +static int +read_interp_state_and_maybe_thread_frame( + RemoteUnwinderObject *unwinder, + uintptr_t interpreter_addr, + char *interp_state_buffer, + char *tstate_buffer, + char *frame_buffer, + RemoteReadPrefetch *prefetch) +{ + prefetch->tstate = NULL; + prefetch->frame = NULL; + if (prefetch->tstate_addr != 0) { + size_t tstate_size = (size_t)unwinder->debug_offsets.thread_state.size; + _Py_RemoteReadSegment segments[3] = { + {interpreter_addr, interp_state_buffer, INTERP_STATE_BUFFER_SIZE}, + {prefetch->tstate_addr, tstate_buffer, tstate_size}, + {prefetch->frame_addr, frame_buffer, SIZEOF_INTERP_FRAME}, + }; + int nsegs = prefetch->frame_addr != 0 ? 3 : 2; + Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory( + &unwinder->handle, segments, nsegs); + int completed = 0; + if (nread >= (Py_ssize_t)INTERP_STATE_BUFFER_SIZE) { + completed = 1; + Py_ssize_t with_tstate = (Py_ssize_t)INTERP_STATE_BUFFER_SIZE + + (Py_ssize_t)tstate_size; + if (nread >= with_tstate) { + completed = 2; + } + if (nsegs == 3 + && nread == with_tstate + (Py_ssize_t)SIZEOF_INTERP_FRAME) { + completed = 3; + } + } + STATS_BATCHED_READ(unwinder, nsegs, completed); + if (completed >= 1) { + if (completed >= 2) { + prefetch->tstate = tstate_buffer; + } + if (completed >= 3) { + prefetch->frame = frame_buffer; + } + return 0; + } + } + return _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer); +} + /*[clinic input] -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_stack_trace Returns stack traces for all interpreters and threads in process. -Each element in the returned list is a tuple of (interpreter_id, thread_list), where: +Each element in the returned list is a tuple of (interpreter_id, +thread_list), where: - interpreter_id is the interpreter identifier -- thread_list is a list of tuples (thread_id, frame_list) for threads in that interpreter +- thread_list is a list of tuples (thread_id, frame_list) for + threads in that interpreter - thread_id is the OS thread identifier - - frame_list is a list of tuples (function_name, filename, line_number) representing - the Python stack frames for that thread, ordered from most recent to oldest + - frame_list is a list of tuples (function_name, filename, + line_number) representing the Python stack frames for that + thread, ordered from most recent to oldest The threads returned depend on the initialization parameters: -- If only_active_thread was True: returns only the thread holding the GIL across all interpreters -- If all_threads was True: returns all threads across all interpreters +- If only_active_thread was True: returns only the thread holding + the GIL across all interpreters +- If all_threads was True: returns all threads across all + interpreters - Otherwise: returns only the main thread of each interpreter Example: @@ -512,16 +694,18 @@ The threads returned depend on the initialization parameters: ] Raises: - RuntimeError: If there is an error copying memory from the target process + RuntimeError: If there is an error copying memory from the + target process OSError: If there is an error accessing the target process PermissionError: If access to the target process is denied - UnicodeDecodeError: If there is an error decoding strings from the target process + UnicodeDecodeError: If there is an error decoding strings from + the target process [clinic start generated code]*/ static PyObject * _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=666192b90c69d567 input=bcff01c73cccc1c0]*/ +/*[clinic end generated code: output=666192b90c69d567 input=86a992b853f48aa9]*/ { STATS_INC(self, total_samples); @@ -536,15 +720,32 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self while (current_interpreter != 0) { // Read interpreter state to get the interpreter ID char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; - if (_Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, + char prefetched_tstate[SIZEOF_THREAD_STATE]; + char prefetched_frame[SIZEOF_INTERP_FRAME]; + RemoteReadPrefetch prefetch = {0}; + if (self->cache_frames) { + prefetch.tstate_addr = get_cached_tstate_for_interpreter( + self, current_interpreter); + } + if (prefetch.tstate_addr != 0) { + FrameCacheEntry *entry = frame_cache_find_by_tstate(self, prefetch.tstate_addr); + if (entry && entry->num_addrs > 0) { + prefetch.frame_addr = entry->addrs[0]; + } + } + + if (read_interp_state_and_maybe_thread_frame( + self, current_interpreter, - INTERP_STATE_BUFFER_SIZE, - interp_state_buffer) < 0) { + interp_state_buffer, + prefetched_tstate, + prefetched_frame, + &prefetch) < 0) { set_exception_cause(self, PyExc_RuntimeError, "Failed to read interpreter state buffer"); Py_CLEAR(result); goto exit; } + refresh_generation_caches_from_interp_state(self, current_interpreter, interp_state_buffer); uintptr_t gc_frame = 0; if (self->gc) { @@ -556,25 +757,6 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self int64_t interpreter_id = GET_MEMBER(int64_t, interp_state_buffer, self->debug_offsets.interpreter_state.id); - // Get code object generation from buffer - uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, - self->debug_offsets.interpreter_state.code_object_generation); - - if (code_object_generation != self->code_object_generation) { - self->code_object_generation = code_object_generation; - _Py_hashtable_clear(self->code_object_cache); - } - -#ifdef Py_GIL_DISABLED - // Check TLBC generation and invalidate cache if needed - uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, - self->debug_offsets.interpreter_state.tlbc_generation); - if (current_tlbc_generation != self->tlbc_generation) { - self->tlbc_generation = current_tlbc_generation; - _Py_hashtable_clear(self->tlbc_cache); - } -#endif - // Create a list to hold threads for this interpreter PyObject *interpreter_threads = PyList_New(0); if (!interpreter_threads) { @@ -610,6 +792,9 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self // Target specific thread (only process first interpreter) current_tstate = self->tstate_addr; } + if (current_tstate != 0 && self->cache_frames) { + set_cached_tstate_for_interpreter(self, current_interpreter, current_tstate); + } // Acquire main thread state information uintptr_t main_thread_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, @@ -620,7 +805,8 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self PyObject* frame_info = unwind_stack_for_thread(self, &current_tstate, gil_holder_tstate, gc_frame, - main_thread_tstate); + main_thread_tstate, + &prefetch); if (!frame_info) { // Check if this was an intentional skip due to mode-based filtering if ((self->mode == PROFILING_MODE_CPU || self->mode == PROFILING_MODE_GIL || @@ -719,26 +905,30 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_all_awaited_by Get all tasks and their awaited_by relationships from the remote process. -This provides a tree structure showing which tasks are waiting for other tasks. +This provides a tree structure showing which tasks are waiting for +other tasks. For each task, returns: -1. The call stack frames leading to where the task is currently executing +1. The call stack frames leading to where the task is currently + executing 2. The name of the task -3. A list of tasks that this task is waiting for, with their own frames/names/etc +3. A list of tasks that this task is waiting for, with their own + frames/names/etc Returns a list of [frames, task_name, subtasks] where: -- frames: List of (func_name, filename, lineno) showing the call stack +- frames: List of (func_name, filename, lineno) showing the call + stack - task_name: String identifier for the task - subtasks: List of tasks being awaited by this task, in same format Raises: - RuntimeError: If AsyncioDebug section is not available in the remote process + RuntimeError: If AsyncioDebug section is not available in the + remote process MemoryError: If memory allocation fails OSError: If reading from the remote process fails @@ -765,11 +955,14 @@ Example output: static PyObject * _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6a49cd345e8aec53 input=307f754cbe38250c]*/ +/*[clinic end generated code: output=6a49cd345e8aec53 input=c22bfee0612e0b69]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL; } + if (refresh_generation_caches_for_interpreter(self, self->interpreter_addr) < 0) { + return NULL; + } PyObject *result = PyList_New(0); if (result == NULL) { @@ -807,20 +1000,21 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_async_stack_trace Get the currently running async tasks and their dependency graphs from the remote process. -This returns information about running tasks and all tasks that are waiting for them, -forming a complete dependency graph for each thread's active task. +This returns information about running tasks and all tasks that are +waiting for them, forming a complete dependency graph for each +thread's active task. -For each thread with a running task, returns the running task plus all tasks that -transitively depend on it (tasks waiting for the running task, tasks waiting for -those tasks, etc.). +For each thread with a running task, returns the running task plus +all tasks that transitively depend on it (tasks waiting for the +running task, tasks waiting for those tasks, etc.). -Returns a list of per-thread results, where each thread result contains: +Returns a list of per-thread results, where each thread result +contains: - Thread ID - List of task information for the running task and all its waiters @@ -831,11 +1025,13 @@ Each task info contains: - List of tasks waiting for this task (recursive structure) Raises: - RuntimeError: If AsyncioDebug section is not available in the target process + RuntimeError: If AsyncioDebug section is not available in the + target process MemoryError: If memory allocation fails OSError: If reading from the remote process fails -Example output (similar structure to get_all_awaited_by but only for running tasks): +Example output (similar structure to get_all_awaited_by but only for +running tasks): [ # Thread 140234 results (140234, [ @@ -854,11 +1050,14 @@ Example output (similar structure to get_all_awaited_by but only for running tas static PyObject * _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6433d52b55e87bbe input=6129b7d509a887c9]*/ +/*[clinic end generated code: output=6433d52b55e87bbe input=bc802e4221c99399]*/ { if (ensure_async_debug_offsets(self) < 0) { return NULL; } + if (refresh_generation_caches_for_interpreter(self, self->interpreter_addr) < 0) { + return NULL; + } PyObject *result = PyList_New(0); if (result == NULL) { @@ -880,7 +1079,6 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject } /*[clinic input] -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.get_stats @@ -895,16 +1093,34 @@ RemoteUnwinder was created with stats=True. - total_samples: Total number of get_stack_trace calls - frame_cache_hits: Full cache hits (entire stack unchanged) - frame_cache_misses: Cache misses requiring full walk - - frame_cache_partial_hits: Partial hits (stopped at cached frame) + - frame_cache_partial_hits: Partial hits (stopped at cached + frame) - frames_read_from_cache: Total frames retrieved from cache - - frames_read_from_memory: Total frames read from remote memory + - frames_read_from_memory: Total frames read from remote + memory - memory_reads: Total remote memory read operations - memory_bytes_read: Total bytes read from remote memory - code_object_cache_hits: Code object cache hits - code_object_cache_misses: Code object cache misses - - stale_cache_invalidations: Times stale cache entries were cleared - - frame_cache_hit_rate: Percentage of samples that hit the cache - - code_object_cache_hit_rate: Percentage of code object lookups that hit cache + - stale_cache_invalidations: Times stale cache entries were + cleared + - batched_read_attempts: Batched remote-read attempts + - batched_read_successes: Attempts that read all requested + segments + - batched_read_misses: Attempts that fell back or partially + read + - batched_read_segments_requested: Segments requested by + batched reads + - batched_read_segments_completed: Segments completed by + batched reads + - frame_cache_hit_rate: Percentage of samples that hit the + cache + - code_object_cache_hit_rate: Percentage of code object + lookups that hit cache + - batched_read_success_rate: Percentage of batched reads + that completed all segments + - batched_read_segment_completion_rate: Percentage of + requested segments read by batched reads Raises: RuntimeError: If stats collection was not enabled (stats=False) @@ -912,7 +1128,7 @@ RemoteUnwinder was created with stats=True. static PyObject * _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=21e36477122be2a0 input=75fef4134c12a8c9]*/ +/*[clinic end generated code: output=21e36477122be2a0 input=87905c65038fb06e]*/ { if (!self->collect_stats) { PyErr_SetString(PyExc_RuntimeError, @@ -947,9 +1163,24 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) ADD_STAT(code_object_cache_hits); ADD_STAT(code_object_cache_misses); ADD_STAT(stale_cache_invalidations); + ADD_STAT(batched_read_attempts); + ADD_STAT(batched_read_successes); + ADD_STAT(batched_read_misses); + ADD_STAT(batched_read_segments_requested); + ADD_STAT(batched_read_segments_completed); #undef ADD_STAT +#define ADD_DERIVED_STAT(name, value) do { \ + PyObject *val = PyFloat_FromDouble(value); \ + if (!val || PyDict_SetItemString(result, name, val) < 0) { \ + Py_XDECREF(val); \ + Py_DECREF(result); \ + return NULL; \ + } \ + Py_DECREF(val); \ +} while(0) + // Calculate and add derived statistics // Hit rate is calculated as (hits + partial_hits) / total_cache_lookups double frame_cache_hit_rate = 0.0; @@ -958,41 +1189,49 @@ _remote_debugging_RemoteUnwinder_get_stats_impl(RemoteUnwinderObject *self) frame_cache_hit_rate = 100.0 * (double)(self->stats.frame_cache_hits + self->stats.frame_cache_partial_hits) / (double)total_cache_lookups; } - PyObject *hit_rate = PyFloat_FromDouble(frame_cache_hit_rate); - if (!hit_rate || PyDict_SetItemString(result, "frame_cache_hit_rate", hit_rate) < 0) { - Py_XDECREF(hit_rate); - Py_DECREF(result); - return NULL; - } - Py_DECREF(hit_rate); + ADD_DERIVED_STAT("frame_cache_hit_rate", frame_cache_hit_rate); double code_object_hit_rate = 0.0; uint64_t total_code_lookups = self->stats.code_object_cache_hits + self->stats.code_object_cache_misses; if (total_code_lookups > 0) { code_object_hit_rate = 100.0 * (double)self->stats.code_object_cache_hits / (double)total_code_lookups; } - PyObject *code_hit_rate = PyFloat_FromDouble(code_object_hit_rate); - if (!code_hit_rate || PyDict_SetItemString(result, "code_object_cache_hit_rate", code_hit_rate) < 0) { - Py_XDECREF(code_hit_rate); - Py_DECREF(result); - return NULL; + ADD_DERIVED_STAT("code_object_cache_hit_rate", code_object_hit_rate); + + double batched_read_success_rate = 0.0; + if (self->stats.batched_read_attempts > 0) { + batched_read_success_rate = + 100.0 * (double)self->stats.batched_read_successes + / (double)self->stats.batched_read_attempts; } - Py_DECREF(code_hit_rate); + ADD_DERIVED_STAT("batched_read_success_rate", batched_read_success_rate); + + double batched_read_segment_completion_rate = 0.0; + if (self->stats.batched_read_segments_requested > 0) { + batched_read_segment_completion_rate = + 100.0 * (double)self->stats.batched_read_segments_completed + / (double)self->stats.batched_read_segments_requested; + } + ADD_DERIVED_STAT("batched_read_segment_completion_rate", + batched_read_segment_completion_rate); + +#undef ADD_DERIVED_STAT return result; } /*[clinic input] -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.pause_threads Pause all threads in the target process. This stops all threads in the target process to allow for consistent -memory reads during sampling. Must be paired with a call to resume_threads(). +memory reads during sampling. Must be paired with a call to +resume_threads(). -Returns True if threads were successfully paused, False if they were already paused. +Returns True if threads were successfully paused, False if they were +already paused. Raises: RuntimeError: If there is an error stopping the threads @@ -1000,7 +1239,7 @@ Returns True if threads were successfully paused, False if they were already pau static PyObject * _remote_debugging_RemoteUnwinder_pause_threads_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=aaf2bdc0a725750c input=d8a266f19a81c67e]*/ +/*[clinic end generated code: output=aaf2bdc0a725750c input=b91dde5517c9dde2]*/ { #ifdef Py_REMOTE_DEBUG_SUPPORTS_BLOCKING if (self->threads_stopped) { @@ -1022,20 +1261,21 @@ _remote_debugging_RemoteUnwinder_pause_threads_impl(RemoteUnwinderObject *self) } /*[clinic input] -@permit_long_docstring_body @critical_section _remote_debugging.RemoteUnwinder.resume_threads Resume all threads in the target process. -This resumes threads that were previously paused with pause_threads(). +This resumes threads that were previously paused with +pause_threads(). -Returns True if threads were successfully resumed, False if they were not paused. +Returns True if threads were successfully resumed, False if they +were not paused. [clinic start generated code]*/ static PyObject * _remote_debugging_RemoteUnwinder_resume_threads_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=8d6781ea37095536 input=16baaaab007f4259]*/ +/*[clinic end generated code: output=8d6781ea37095536 input=130758d55d46897a]*/ { #ifdef Py_REMOTE_DEBUG_SUPPORTS_BLOCKING if (!self->threads_stopped) { @@ -1173,7 +1413,6 @@ class _remote_debugging.GCMonitor "GCMonitorObject *" "&GCMonitor_Type" /*[clinic input] @permit_long_summary -@permit_long_docstring_body _remote_debugging.GCMonitor.__init__ pid: int * @@ -1183,22 +1422,24 @@ Initialize a new GCMonitor object for monitoring GC events from remote process. Args: pid: Process ID of the target Python process to monitor - debug: If True, chain exceptions to explain the sequence of events that - lead to the exception. + debug: If True, chain exceptions to explain the sequence of + events that lead to the exception. -The GCMonitor provides functionality to read GC statistics from a running -Python process. +The GCMonitor provides functionality to read GC statistics from +a running Python process. Raises: PermissionError: If access to the target process is denied - OSError: If unable to attach to the target process or access its memory - RuntimeError: If unable to read debug information from the target process + OSError: If unable to attach to the target process or access + its memory + RuntimeError: If unable to read debug information from the + target process [clinic start generated code]*/ static int _remote_debugging_GCMonitor___init___impl(GCMonitorObject *self, int pid, int debug) -/*[clinic end generated code: output=2cdf351c2f6335db input=1185a48535b808be]*/ +/*[clinic end generated code: output=2cdf351c2f6335db input=03da0b2d3282ae1b]*/ { return init_runtime_offsets(&self->offsets, pid, debug); } @@ -1225,17 +1466,18 @@ Returns a list of GCStatsInfo objects with GC statistics data. - collected: Total number of collected objects. - uncollectable: Total number of uncollectable objects. - candidates: Total objects considered and traversed. + - heap_size: number of live objects. - duration: Total collection time, in seconds. Raises: - RuntimeError: If the target process cannot be inspected or if its - debug offsets or GC stats layout are incompatible. + RuntimeError: If the target process cannot be inspected or if + its debug offsets or GC stats layout are incompatible. [clinic start generated code]*/ static PyObject * _remote_debugging_GCMonitor_get_gc_stats_impl(GCMonitorObject *self, int all_interpreters) -/*[clinic end generated code: output=f73f365725224f7a input=09e647719c65f9e4]*/ +/*[clinic end generated code: output=f73f365725224f7a input=ec016bc4be6dd003]*/ { RemoteDebuggingState *st = RemoteDebugging_GetStateFromType(Py_TYPE(self)); return get_gc_stats(&self->offsets, all_interpreters, st->GCStatsInfo_Type); @@ -1382,9 +1624,6 @@ _remote_debugging_exec(PyObject *m) return -1; } -#ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif int rc = PyModule_AddIntConstant(m, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); if (rc < 0) { return -1; @@ -1472,9 +1711,8 @@ class _remote_debugging.BinaryWriter "BinaryWriterObject *" "&PyBinaryWriter_Typ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=e948838b90a2003c]*/ /*[clinic input] -@permit_long_docstring_body _remote_debugging.BinaryWriter.__init__ - filename: str + filename: object sample_interval_us: unsigned_long_long start_time_us: unsigned_long_long * @@ -1485,7 +1723,8 @@ High-performance binary writer for profiling data. Arguments: filename: Path to output file sample_interval_us: Sampling interval in microseconds - start_time_us: Start timestamp in microseconds (from time.monotonic() * 1e6) + start_time_us: Start timestamp in microseconds (from + time.monotonic() * 1e6) compression: 0=none, 1=zstd (default: 0) Use as a context manager or call finalize() when done. @@ -1493,11 +1732,11 @@ Use as a context manager or call finalize() when done. static int _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, - const char *filename, + PyObject *filename, unsigned long long sample_interval_us, unsigned long long start_time_us, int compression) -/*[clinic end generated code: output=014c0306f1bacf4b input=3bdf01c1cc2f5a1d]*/ +/*[clinic end generated code: output=00446656ea2e5986 input=2e3f298c69fc7666]*/ { if (self->writer) { binary_writer_destroy(self->writer); @@ -1512,7 +1751,6 @@ _remote_debugging_BinaryWriter___init___impl(BinaryWriterObject *self, } /*[clinic input] -@permit_long_docstring_body _remote_debugging.BinaryWriter.write_sample stack_frames: object timestamp_us: unsigned_long_long @@ -1521,14 +1759,15 @@ Write a sample to the binary file. Arguments: stack_frames: List of InterpreterInfo objects - timestamp_us: Current timestamp in microseconds (from time.monotonic() * 1e6) + timestamp_us: Current timestamp in microseconds (from + time.monotonic() * 1e6) [clinic start generated code]*/ static PyObject * _remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self, PyObject *stack_frames, unsigned long long timestamp_us) -/*[clinic end generated code: output=24d5b86679b4128f input=4e6d832d360bea46]*/ +/*[clinic end generated code: output=24d5b86679b4128f input=5033f1ae7fa135f1]*/ { if (!self->writer) { PyErr_SetString(PyExc_ValueError, "Writer is closed"); @@ -1542,6 +1781,24 @@ _remote_debugging_BinaryWriter_write_sample_impl(BinaryWriterObject *self, Py_RETURN_NONE; } +/* Finalize the writer, cache total_samples, and destroy it. + * + * The cache assignment must happen AFTER binary_writer_finalize(): finalize + * flushes pending RLE samples via flush_pending_rle(), which increments + * writer->total_samples for each one. Caching before finalize would lose + * those trailing samples. */ +static int +binary_writer_finalize_and_cache(BinaryWriterObject *self) +{ + if (binary_writer_finalize(self->writer) < 0) { + return -1; + } + self->cached_total_samples = self->writer->total_samples; + binary_writer_destroy(self->writer); + self->writer = NULL; + return 0; +} + /*[clinic input] _remote_debugging.BinaryWriter.finalize @@ -1559,16 +1816,10 @@ _remote_debugging_BinaryWriter_finalize_impl(BinaryWriterObject *self) return NULL; } - /* Save total_samples before finalizing */ - self->cached_total_samples = self->writer->total_samples; - - if (binary_writer_finalize(self->writer) < 0) { + if (binary_writer_finalize_and_cache(self) < 0) { return NULL; } - binary_writer_destroy(self->writer); - self->writer = NULL; - Py_RETURN_NONE; } @@ -1622,31 +1873,35 @@ _remote_debugging_BinaryWriter___exit___impl(BinaryWriterObject *self, if (self->writer) { /* Only finalize on normal exit (no exception) */ if (exc_type == Py_None) { - if (binary_writer_finalize(self->writer) < 0) { - binary_writer_destroy(self->writer); - self->writer = NULL; + if (binary_writer_finalize_and_cache(self) < 0) { + if (self->writer) { + binary_writer_destroy(self->writer); + self->writer = NULL; + } return NULL; } } - binary_writer_destroy(self->writer); - self->writer = NULL; + else { + binary_writer_destroy(self->writer); + self->writer = NULL; + } } Py_RETURN_FALSE; } /*[clinic input] -@permit_long_docstring_body _remote_debugging.BinaryWriter.get_stats Get encoding statistics for the writer. -Returns a dict with encoding statistics including repeat/full/suffix/pop-push -record counts, frames written/saved, and compression ratio. +Returns a dict with encoding statistics including +repeat/full/suffix/pop-push record counts, frames written/saved, and +compression ratio. [clinic start generated code]*/ static PyObject * _remote_debugging_BinaryWriter_get_stats_impl(BinaryWriterObject *self) -/*[clinic end generated code: output=06522cd52544df89 input=837c874ffdebd24c]*/ +/*[clinic end generated code: output=06522cd52544df89 input=a8bb8c8682ccd34b]*/ { if (!self->writer) { PyErr_SetString(PyExc_ValueError, "Writer is closed"); @@ -1656,8 +1911,9 @@ _remote_debugging_BinaryWriter_get_stats_impl(BinaryWriterObject *self) } static PyObject * -BinaryWriter_get_total_samples(BinaryWriterObject *self, void *closure) +BinaryWriter_get_total_samples(PyObject *op, void *closure) { + BinaryWriterObject *self = BinaryWriter_CAST(op); if (!self->writer) { /* Use cached value after finalize/close */ return PyLong_FromUnsignedLong(self->cached_total_samples); @@ -1666,7 +1922,7 @@ BinaryWriter_get_total_samples(BinaryWriterObject *self, void *closure) } static PyGetSetDef BinaryWriter_getset[] = { - {"total_samples", (getter)BinaryWriter_get_total_samples, NULL, "Total samples written", NULL}, + {"total_samples", BinaryWriter_get_total_samples, NULL, "Total samples written", NULL}, {NULL} }; @@ -1723,7 +1979,7 @@ class _remote_debugging.BinaryReader "BinaryReaderObject *" "&PyBinaryReader_Typ /*[clinic input] _remote_debugging.BinaryReader.__init__ - filename: str + filename: object High-performance binary reader for profiling data. @@ -1735,8 +1991,8 @@ Use as a context manager or call close() when done. static int _remote_debugging_BinaryReader___init___impl(BinaryReaderObject *self, - const char *filename) -/*[clinic end generated code: output=9699226f7ae052bb input=4201f9cc500ef2f6]*/ + PyObject *filename) +/*[clinic end generated code: output=f04b33ee5c5e6dbf input=9d7cbe8b4f1a97c9]*/ { if (self->reader) { binary_reader_close(self->reader); @@ -1810,13 +2066,13 @@ _remote_debugging.BinaryReader.get_stats Get reconstruction statistics from replay. -Returns a dict with statistics about record types decoded and samples -reconstructed during replay. +Returns a dict with statistics about record types decoded and +samples reconstructed during replay. [clinic start generated code]*/ static PyObject * _remote_debugging_BinaryReader_get_stats_impl(BinaryReaderObject *self) -/*[clinic end generated code: output=628b9ab5e4c4fd36 input=d8dd6654abd6c3c0]*/ +/*[clinic end generated code: output=628b9ab5e4c4fd36 input=15b8d8f89ccf3726]*/ { if (!self->reader) { PyErr_SetString(PyExc_ValueError, "Reader is closed"); @@ -1968,7 +2224,6 @@ _remote_debugging_zstd_available_impl(PyObject *module) * ============================================================================ */ /*[clinic input] -@permit_long_docstring_body _remote_debugging.get_child_pids pid: int @@ -1980,11 +2235,12 @@ _remote_debugging.get_child_pids Get all child process IDs of the given process. -Returns a list of child process IDs. Returns an empty list if no children -are found. +Returns a list of child process IDs. Returns an empty list if no +children are found. -This function provides a snapshot of child processes at a moment in time. -Child processes may exit or new ones may be created after the list is returned. +This function provides a snapshot of child processes at a moment in +time. Child processes may exit or new ones may be created after the +list is returned. Raises: OSError: If unable to enumerate processes @@ -1994,7 +2250,7 @@ Child processes may exit or new ones may be created after the list is returned. static PyObject * _remote_debugging_get_child_pids_impl(PyObject *module, int pid, int recursive) -/*[clinic end generated code: output=1ae2289c6b953e4b input=19d8d5d6e2b59e6e]*/ +/*[clinic end generated code: output=1ae2289c6b953e4b input=c6437b52e2fdd880]*/ { return enumerate_child_pids((pid_t)pid, recursive); } diff --git a/Modules/_remote_debugging/object_reading.c b/Modules/_remote_debugging/object_reading.c index b63b103a2617acc..1cea96a2151fcc6 100644 --- a/Modules/_remote_debugging/object_reading.c +++ b/Modules/_remote_debugging/object_reading.c @@ -6,6 +6,7 @@ ******************************************************************************/ #include "_remote_debugging.h" +#include <limits.h> /* ============================================================================ * MEMORY READING FUNCTIONS @@ -264,26 +265,16 @@ read_py_long( Py_ssize_t inline_digits_space = SIZEOF_LONG_OBJ - ob_digit_offset; Py_ssize_t max_inline_digits = inline_digits_space / (Py_ssize_t)sizeof(digit); - // If the long object has inline digits that fit in our buffer, use them directly - digit *digits; + digit *digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for PyLong"); + return -1; + } + if (size <= max_inline_digits && size <= _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS) { - // For small integers, digits are inline in the long_value.ob_digit array - digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); - if (!digits) { - PyErr_NoMemory(); - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for small PyLong"); - return -1; - } memcpy(digits, long_obj + ob_digit_offset, size * sizeof(digit)); } else { - // For larger integers, we need to read the digits separately - digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); - if (!digits) { - PyErr_NoMemory(); - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for large PyLong"); - return -1; - } - bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, address + (uintptr_t)unwinder->debug_offsets.long_object.ob_digit, @@ -296,19 +287,34 @@ read_py_long( } } - long long value = 0; + unsigned long limit = negative + ? (unsigned long)LONG_MAX + 1UL + : (unsigned long)LONG_MAX; + unsigned long value = 0; - // In theory this can overflow, but because of llvm/llvm-project#16778 - // we can't use __builtin_mul_overflow because it fails to link with - // __muloti4 on aarch64. In practice this is fine because all we're - // testing here are task numbers that would fit in a single byte. - for (Py_ssize_t i = 0; i < size; ++i) { - long long factor = digits[i] * (1UL << (Py_ssize_t)(shift * i)); - value += factor; + for (Py_ssize_t i = size; i-- > 0;) { + if (digits[i] >= PyLong_BASE) { + PyErr_Format(PyExc_RuntimeError, + "Invalid PyLong digit: %u (base %u)", digits[i], PyLong_BASE); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid PyLong digit (corrupted remote memory)"); + goto error; + } + if (value > ((limit - (unsigned long)digits[i]) >> shift)) { + PyErr_SetString(PyExc_OverflowError, + "Remote PyLong value does not fit in C long"); + set_exception_cause(unwinder, PyExc_OverflowError, + "Remote PyLong value is too large"); + goto error; + } + value = (value << shift) | (unsigned long)digits[i]; } PyMem_RawFree(digits); if (negative) { - value *= -1; + if (value == (unsigned long)LONG_MAX + 1UL) { + return LONG_MIN; + } + return -(long)value; } return (long)value; error: diff --git a/Modules/_remote_debugging/subprocess.c b/Modules/_remote_debugging/subprocess.c index 1b16dd8343f2a5b..cdad75e318be914 100644 --- a/Modules/_remote_debugging/subprocess.c +++ b/Modules/_remote_debugging/subprocess.c @@ -223,8 +223,19 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) } /* Single pass: collect PIDs and their PPIDs together */ - struct dirent *entry; - while ((entry = readdir(proc_dir)) != NULL) { + for (;;) { + errno = 0; + struct dirent *entry = readdir(proc_dir); + if (entry == NULL) { + if (errno != 0) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, "/proc", + "Failed to read process directory '/proc': %s", + strerror(err)); + goto done; + } + break; + } /* Skip non-numeric entries (also skips . and ..) */ if (entry->d_name[0] < '1' || entry->d_name[0] > '9') { continue; @@ -245,7 +256,14 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) } } - closedir(proc_dir); + if (closedir(proc_dir) != 0) { + int err = errno; + proc_dir = NULL; + _set_debug_oserror_from_errno_with_filename(err, "/proc", + "Failed to close process directory '/proc': %s", + strerror(err)); + goto done; + } proc_dir = NULL; if (find_children_bfs(target_pid, recursive, @@ -358,7 +376,8 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); if (snapshot == INVALID_HANDLE_VALUE) { - PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); goto done; } @@ -373,13 +392,23 @@ get_child_pids_platform(pid_t target_pid, int recursive, pid_array_t *result) /* Single pass: collect PIDs and PPIDs together */ PROCESSENTRY32 pe; pe.dwSize = sizeof(PROCESSENTRY32); - if (Process32First(snapshot, &pe)) { - do { - if (pid_array_append(&all_pids, (pid_t)pe.th32ProcessID) < 0 || - pid_array_append(&ppids, (pid_t)pe.th32ParentProcessID) < 0) { - goto done; - } - } while (Process32Next(snapshot, &pe)); + if (!Process32First(snapshot, &pe)) { + DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); + goto done; + } + + do { + if (pid_array_append(&all_pids, (pid_t)pe.th32ProcessID) < 0 || + pid_array_append(&ppids, (pid_t)pe.th32ParentProcessID) < 0) { + goto done; + } + } while (Process32Next(snapshot, &pe)); + + DWORD error = GetLastError(); + if (error != ERROR_NO_MORE_FILES) { + PyErr_SetFromWindowsErr(error); + goto done; } CloseHandle(snapshot); diff --git a/Modules/_remote_debugging/threads.c b/Modules/_remote_debugging/threads.c index e303c667ea013a8..81735e85395ac9e 100644 --- a/Modules/_remote_debugging/threads.c +++ b/Modules/_remote_debugging/threads.c @@ -34,11 +34,11 @@ iterate_threads( if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, - unwinder->interpreter_addr + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main, + unwinder->interpreter_addr + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_head, sizeof(void*), &thread_state_addr)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read main thread state"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read threads head"); return -1; } @@ -289,28 +289,110 @@ typedef struct { unsigned int :24; } _thread_status; +static int +read_thread_state_and_maybe_frame( + RemoteUnwinderObject *unwinder, + uintptr_t tstate_addr, + size_t tstate_size, + char *tstate_buffer, + uintptr_t predicted_frame_addr, + char *frame_buffer, + int *frame_read) +{ + *frame_read = 0; + if (predicted_frame_addr != 0) { + _Py_RemoteReadSegment segments[2] = { + {tstate_addr, tstate_buffer, tstate_size}, + {predicted_frame_addr, frame_buffer, SIZEOF_INTERP_FRAME}, + }; + Py_ssize_t nread = _Py_RemoteDebug_BatchedReadRemoteMemory( + &unwinder->handle, segments, 2); + int completed = 0; + if (nread >= (Py_ssize_t)tstate_size) { + completed = 1; + if (nread == (Py_ssize_t)(tstate_size + SIZEOF_INTERP_FRAME)) { + completed = 2; + } + } + STATS_BATCHED_READ(unwinder, 2, completed); + if (completed >= 1) { + *frame_read = completed == 2; + return 0; + } + } + return _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, tstate_addr, tstate_size, tstate_buffer); +} + PyObject* unwind_stack_for_thread( RemoteUnwinderObject *unwinder, uintptr_t *current_tstate, uintptr_t gil_holder_tstate, uintptr_t gc_frame, - uintptr_t main_thread_tstate + uintptr_t main_thread_tstate, + const RemoteReadPrefetch *prefetch ) { PyObject *frame_info = NULL; PyObject *thread_id = NULL; PyObject *result = NULL; StackChunkList chunks = {0}; - char ts[SIZEOF_THREAD_STATE]; - int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, *current_tstate, (size_t)unwinder->debug_offsets.thread_state.size, ts); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); - goto error; + char local_ts[SIZEOF_THREAD_STATE]; + char local_prefetched_frame[SIZEOF_INTERP_FRAME]; + const char *ts; + RemoteReadPrefetch ctx_prefetch = {0}; + if (prefetch->tstate && prefetch->tstate_addr == *current_tstate) { + ts = prefetch->tstate; + if (prefetch->frame) { + ctx_prefetch.frame = prefetch->frame; + ctx_prefetch.frame_addr = prefetch->frame_addr; + } + } + else if (unwinder->cache_frames) { + uintptr_t predicted_frame_addr = 0; + int have_prefetched_frame = 0; + FrameCacheEntry *entry = frame_cache_find_by_tstate(unwinder, *current_tstate); + if (entry && entry->num_addrs > 0) { + predicted_frame_addr = entry->addrs[0]; + } + + int rc = read_thread_state_and_maybe_frame( + unwinder, + *current_tstate, + (size_t)unwinder->debug_offsets.thread_state.size, + local_ts, + predicted_frame_addr, + local_prefetched_frame, + &have_prefetched_frame); + if (rc < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + ts = local_ts; + if (have_prefetched_frame) { + ctx_prefetch.frame = local_prefetched_frame; + ctx_prefetch.frame_addr = predicted_frame_addr; + } + } + else { + int rc = _Py_RemoteDebug_ReadRemoteMemory( + &unwinder->handle, + *current_tstate, + (size_t)unwinder->debug_offsets.thread_state.size, + local_ts); + if (rc < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + ts = local_ts; } STATS_INC(unwinder, memory_reads); STATS_ADD(unwinder, memory_bytes_read, unwinder->debug_offsets.thread_state.size); + if (ctx_prefetch.frame) { + STATS_INC(unwinder, memory_reads); + STATS_ADD(unwinder, memory_bytes_read, SIZEOF_INTERP_FRAME); + } long tid = GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id); @@ -432,9 +514,11 @@ unwind_stack_for_thread( uintptr_t addrs[FRAME_CACHE_MAX_FRAMES]; FrameWalkContext ctx = { .frame_addr = frame_addr, + .thread_state_addr = *current_tstate, .base_frame_addr = base_frame_addr, .gc_frame = gc_frame, .chunks = &chunks, + .prefetch = ctx_prefetch, .frame_info = frame_info, .frame_addrs = addrs, .num_addrs = 0, @@ -450,12 +534,14 @@ unwind_stack_for_thread( set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to collect frames"); goto error; } - // Update last_profiled_frame for next sample - uintptr_t lpf_addr = - *current_tstate + (uintptr_t)unwinder->debug_offsets.thread_state.last_profiled_frame; - if (_Py_RemoteDebug_WriteRemoteMemory(&unwinder->handle, lpf_addr, - sizeof(uintptr_t), &frame_addr) < 0) { - PyErr_Clear(); // Non-fatal + // Update last_profiled_frame for next sample if it changed + if (frame_addr != ctx.last_profiled_frame) { + uintptr_t lpf_addr = + *current_tstate + (uintptr_t)unwinder->debug_offsets.thread_state.last_profiled_frame; + if (_Py_RemoteDebug_WriteRemoteMemory(&unwinder->handle, lpf_addr, + sizeof(uintptr_t), &frame_addr) < 0) { + PyErr_Clear(); // Non-fatal + } } } else { // No caching - process entire frame chain with base_frame validation @@ -467,10 +553,18 @@ unwind_stack_for_thread( *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); - thread_id = PyLong_FromLongLong(tid); + if (unwinder->cache_frames) { + FrameCacheEntry *entry = frame_cache_find(unwinder, (uint64_t)tid); + if (entry && entry->thread_id_obj) { + thread_id = Py_NewRef(entry->thread_id_obj); + } + } if (thread_id == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); - goto error; + thread_id = PyLong_FromLongLong(tid); + if (thread_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + goto error; + } } RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); @@ -566,8 +660,7 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st DIR *dir = opendir(task_path); if (dir == NULL) { - st->tids = NULL; - st->count = 0; + _Py_RemoteDebug_InitThreadsState(unwinder, st); if (errno == ENOENT || errno == ESRCH) { PyErr_Format(PyExc_ProcessLookupError, "Process %d has terminated", unwinder->handle.pid); @@ -579,8 +672,21 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st st->count = 0; - struct dirent *entry; - while ((entry = readdir(dir)) != NULL) { + for (;;) { + errno = 0; + struct dirent *entry = readdir(dir); + if (entry == NULL) { + if (errno != 0) { + int err = errno; + closedir(dir); + _Py_RemoteDebug_InitThreadsState(unwinder, st); + _set_debug_oserror_from_errno_with_filename(err, task_path, + "Failed to read process task directory '%s': %s", + task_path, strerror(err)); + return -1; + } + break; + } if (entry->d_name[0] < '1' || entry->d_name[0] > '9') { continue; } @@ -594,8 +700,7 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st pid_t *new_tids = PyMem_RawRealloc(unwinder->thread_tids, new_cap * sizeof(pid_t)); if (new_tids == NULL) { closedir(dir); - st->tids = NULL; - st->count = 0; + _Py_RemoteDebug_InitThreadsState(unwinder, st); PyErr_NoMemory(); return -1; } @@ -605,8 +710,15 @@ read_thread_ids(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_ThreadsState *st unwinder->thread_tids[st->count++] = (pid_t)tid; } + if (closedir(dir) != 0) { + int err = errno; + _Py_RemoteDebug_InitThreadsState(unwinder, st); + _set_debug_oserror_from_errno_with_filename(err, task_path, + "Failed to close process task directory '%s': %s", + task_path, strerror(err)); + return -1; + } st->tids = unwinder->thread_tids; - closedir(dir); return 0; } @@ -619,28 +731,30 @@ detach_threads(_Py_RemoteDebug_ThreadsState *st, size_t up_to) } static int -seize_thread(pid_t tid) +seize_thread(pid_t tid, int *err) { if (ptrace(PTRACE_SEIZE, tid, NULL, 0) == 0) { return 0; } - if (errno == ESRCH) { + *err = errno; + if (*err == ESRCH) { return 1; // Thread gone, skip } - if (errno == EPERM) { + if (*err == EPERM) { // Thread may have exited, be in a special state, or already be traced. // Skip rather than fail - this avoids endless retry loops when // threads transiently become inaccessible. return 1; } - if (errno == EINVAL || errno == EIO) { + if (*err == EINVAL || *err == EIO) { // Fallback for older kernels if (ptrace(PTRACE_ATTACH, tid, NULL, NULL) == 0) { int status; waitpid(tid, &status, __WALL); return 0; } - if (errno == ESRCH || errno == EPERM) { + *err = errno; + if (*err == ESRCH || *err == EPERM) { return 1; // Thread gone or inaccessible } } @@ -654,39 +768,50 @@ _Py_RemoteDebug_StopAllThreads(RemoteUnwinderObject *unwinder, _Py_RemoteDebug_T return -1; } - for (size_t i = 0; i < st->count; i++) { + size_t n_tids = st->count; + size_t seized = 0; + for (size_t i = 0; i < n_tids; i++) { pid_t tid = st->tids[i]; - int ret = seize_thread(tid); + int err = 0; + int ret = seize_thread(tid, &err); if (ret == 1) { continue; // Thread gone, skip } if (ret < 0) { - detach_threads(st, i); - PyErr_Format(PyExc_RuntimeError, "Failed to seize thread %d: %s", tid, strerror(errno)); - st->tids = NULL; - st->count = 0; + detach_threads(st, seized); + _set_debug_oserror_from_errno(err, + "Failed to seize thread %d: %s", tid, strerror(err)); + _Py_RemoteDebug_InitThreadsState(unwinder, st); return -1; } - - if (ptrace(PTRACE_INTERRUPT, tid, NULL, NULL) == -1 && errno != ESRCH) { - detach_threads(st, i + 1); - PyErr_Format(PyExc_RuntimeError, "Failed to interrupt thread %d: %s", tid, strerror(errno)); - st->tids = NULL; - st->count = 0; - return -1; + st->tids[seized++] = tid; + + if (ptrace(PTRACE_INTERRUPT, tid, NULL, NULL) == -1) { + err = errno; + if (err != ESRCH) { + detach_threads(st, seized); + _set_debug_oserror_from_errno(err, + "Failed to interrupt thread %d: %s", tid, strerror(err)); + _Py_RemoteDebug_InitThreadsState(unwinder, st); + return -1; + } } int status; - if (waitpid(tid, &status, __WALL) == -1 && errno != ECHILD && errno != ESRCH) { - detach_threads(st, i + 1); - PyErr_Format(PyExc_RuntimeError, "waitpid failed for thread %d: %s", tid, strerror(errno)); - st->tids = NULL; - st->count = 0; - return -1; + if (waitpid(tid, &status, __WALL) == -1) { + err = errno; + if (err != ECHILD && err != ESRCH) { + detach_threads(st, seized); + _set_debug_oserror_from_errno(err, + "waitpid failed for thread %d: %s", tid, strerror(err)); + _Py_RemoteDebug_InitThreadsState(unwinder, st); + return -1; + } } } + st->count = seized; return 0; } diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 8dad94556236bd6..d81784409e5d91a 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -166,7 +166,6 @@ read_multiple(pysqlite_Blob *self, Py_ssize_t length, Py_ssize_t offset) /*[clinic input] -@permit_long_docstring_body _sqlite3.Blob.read as blob_read length: int = -1 @@ -175,14 +174,14 @@ _sqlite3.Blob.read as blob_read Read data at the current offset position. -If the end of the blob is reached, the data up to end of file will be returned. -When length is not specified, or is negative, Blob.read() will read until the -end of the blob. +If the end of the blob is reached, the data up to end of file will +be returned. When length is not specified, or is negative, +Blob.read() will read until the end of the blob. [clinic start generated code]*/ static PyObject * blob_read_impl(pysqlite_Blob *self, int length) -/*[clinic end generated code: output=1fc99b2541360dde input=e5715bcddbcfca5a]*/ +/*[clinic end generated code: output=1fc99b2541360dde input=6b745ad37720e556]*/ { if (!check_blob(self)) { return NULL; @@ -235,7 +234,6 @@ inner_write(pysqlite_Blob *self, const void *buf, Py_ssize_t len, /*[clinic input] -@permit_long_docstring_body _sqlite3.Blob.write as blob_write data: Py_buffer @@ -243,13 +241,13 @@ _sqlite3.Blob.write as blob_write Write data at the current offset. -This function cannot change the blob length. Writing beyond the end of the -blob will result in an exception being raised. +This function cannot change the blob length. Writing beyond the end +of the blob will result in an exception being raised. [clinic start generated code]*/ static PyObject * blob_write_impl(pysqlite_Blob *self, Py_buffer *data) -/*[clinic end generated code: output=b34cf22601b570b2 input=203d3458f244814b]*/ +/*[clinic end generated code: output=b34cf22601b570b2 input=0d372cb0240a5d49]*/ { if (!check_blob(self)) { return NULL; @@ -265,7 +263,6 @@ blob_write_impl(pysqlite_Blob *self, Py_buffer *data) /*[clinic input] -@permit_long_docstring_body _sqlite3.Blob.seek as blob_seek offset: int @@ -274,14 +271,15 @@ _sqlite3.Blob.seek as blob_seek Set the current access position to offset. -The origin argument defaults to os.SEEK_SET (absolute blob positioning). -Other values for origin are os.SEEK_CUR (seek relative to the current position) -and os.SEEK_END (seek relative to the blob's end). +The origin argument defaults to os.SEEK_SET (absolute blob +positioning). Other values for origin are os.SEEK_CUR (seek +relative to the current position) and os.SEEK_END (seek relative to +the blob's end). [clinic start generated code]*/ static PyObject * blob_seek_impl(pysqlite_Blob *self, int offset, int origin) -/*[clinic end generated code: output=854c5a0e208547a5 input=ee4d88e1dc0b1048]*/ +/*[clinic end generated code: output=854c5a0e208547a5 input=84aea1b6b48607dd]*/ { if (!check_blob(self)) { return NULL; @@ -533,21 +531,25 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } - if (len == 0) { - return 0; - } - Py_buffer vbuf; if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { return -1; } - int rc = -1; if (vbuf.len != len) { PyErr_SetString(PyExc_IndexError, "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; } - else if (step == 1) { + + if (len == 0) { + PyBuffer_Release(&vbuf); + return 0; + } + + int rc = -1; + if (step == 1) { rc = inner_write(self, vbuf.buf, len, start); } else { diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index 921e7cbd7ffcaba..929703257f04bee 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -31,9 +31,9 @@ PyDoc_STRVAR(blob_read__doc__, " length\n" " Read length in bytes.\n" "\n" -"If the end of the blob is reached, the data up to end of file will be returned.\n" -"When length is not specified, or is negative, Blob.read() will read until the\n" -"end of the blob."); +"If the end of the blob is reached, the data up to end of file will\n" +"be returned. When length is not specified, or is negative,\n" +"Blob.read() will read until the end of the blob."); #define BLOB_READ_METHODDEF \ {"read", _PyCFunction_CAST(blob_read), METH_FASTCALL, blob_read__doc__}, @@ -70,8 +70,8 @@ PyDoc_STRVAR(blob_write__doc__, "\n" "Write data at the current offset.\n" "\n" -"This function cannot change the blob length. Writing beyond the end of the\n" -"blob will result in an exception being raised."); +"This function cannot change the blob length. Writing beyond the end\n" +"of the blob will result in an exception being raised."); #define BLOB_WRITE_METHODDEF \ {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, @@ -105,9 +105,10 @@ PyDoc_STRVAR(blob_seek__doc__, "\n" "Set the current access position to offset.\n" "\n" -"The origin argument defaults to os.SEEK_SET (absolute blob positioning).\n" -"Other values for origin are os.SEEK_CUR (seek relative to the current position)\n" -"and os.SEEK_END (seek relative to the blob\'s end)."); +"The origin argument defaults to os.SEEK_SET (absolute blob\n" +"positioning). Other values for origin are os.SEEK_CUR (seek\n" +"relative to the current position) and os.SEEK_END (seek relative to\n" +"the blob\'s end)."); #define BLOB_SEEK_METHODDEF \ {"seek", _PyCFunction_CAST(blob_seek), METH_FASTCALL, blob_seek__doc__}, @@ -211,4 +212,4 @@ blob_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f03f4ba622b67ae0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b0e3d38063739b17 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index abb864eb0307578..b645bf3464bcea1 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -685,13 +685,14 @@ PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, "\n" " progress_handler\n" " A callable that takes no arguments.\n" -" If the callable returns non-zero, the current query is terminated,\n" -" and an exception is raised.\n" +" If the callable returns non-zero, the current query is\n" +" terminated, and an exception is raised.\n" " n\n" " The number of SQLite virtual machine instructions that are\n" " executed between invocations of \'progress_handler\'.\n" "\n" -"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled."); +"If \'progress_handler\' is None or \'n\' is 0, the progress handler is\n" +"disabled."); #define PYSQLITE_CONNECTION_SET_PROGRESS_HANDLER_METHODDEF \ {"set_progress_handler", _PyCFunction_CAST(pysqlite_connection_set_progress_handler), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pysqlite_connection_set_progress_handler__doc__}, @@ -1303,10 +1304,10 @@ PyDoc_STRVAR(serialize__doc__, " name\n" " Which database to serialize.\n" "\n" -"For an ordinary on-disk database file, the serialization is just a copy of the\n" -"disk file. For an in-memory database or a \"temp\" database, the serialization is\n" -"the same sequence of bytes which would be written to disk if that database\n" -"were backed up to disk."); +"For an ordinary on-disk database file, the serialization is just\n" +"a copy of the disk file. For an in-memory database or a \"temp\"\n" +"database, the serialization is the same sequence of bytes which\n" +"would be written to disk if that database were backed up to disk."); #define SERIALIZE_METHODDEF \ {"serialize", _PyCFunction_CAST(serialize), METH_FASTCALL|METH_KEYWORDS, serialize__doc__}, @@ -1392,12 +1393,13 @@ PyDoc_STRVAR(deserialize__doc__, " name\n" " Which database to reopen with the deserialization.\n" "\n" -"The deserialize interface causes the database connection to disconnect from the\n" -"target database, and then reopen it as an in-memory database based on the given\n" -"serialized data.\n" +"The deserialize interface causes the database connection to\n" +"disconnect from the target database, and then reopen it as\n" +"an in-memory database based on the given serialized data.\n" "\n" -"The deserialize interface will fail with SQLITE_BUSY if the database is\n" -"currently in a read transaction or is involved in a backup operation."); +"The deserialize interface will fail with SQLITE_BUSY if the database\n" +"is currently in a read transaction or is involved in a backup\n" +"operation."); #define DESERIALIZE_METHODDEF \ {"deserialize", _PyCFunction_CAST(deserialize), METH_FASTCALL|METH_KEYWORDS, deserialize__doc__}, @@ -1518,7 +1520,8 @@ PyDoc_STRVAR(pysqlite_connection_exit__doc__, "\n" "Called when the connection is used as a context manager.\n" "\n" -"If there was any exception, a rollback takes place; otherwise we commit."); +"If there was any exception, a rollback takes place; otherwise we\n" +"commit."); #define PYSQLITE_CONNECTION_EXIT_METHODDEF \ {"__exit__", _PyCFunction_CAST(pysqlite_connection_exit), METH_FASTCALL, pysqlite_connection_exit__doc__}, @@ -1556,12 +1559,12 @@ PyDoc_STRVAR(setlimit__doc__, " category\n" " The limit category to be set.\n" " limit\n" -" The new limit. If the new limit is a negative number, the limit is\n" -" unchanged.\n" +" The new limit. If the new limit is a negative number, the limit\n" +" is unchanged.\n" "\n" -"Attempts to increase a limit above its hard upper bound are silently truncated\n" -"to the hard upper bound. Regardless of whether or not the limit was changed,\n" -"the prior value of the limit is returned."); +"Attempts to increase a limit above its hard upper bound are silently\n" +"truncated to the hard upper bound. Regardless of whether or not the\n" +"limit was changed, the prior value of the limit is returned."); #define SETLIMIT_METHODDEF \ {"setlimit", _PyCFunction_CAST(setlimit), METH_FASTCALL, setlimit__doc__}, @@ -1722,4 +1725,4 @@ getconfig(PyObject *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=16d44c1d8a45e622 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1418b72751ef68fc input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index bd44ff31b87c67b..892740b05e55c98 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -557,6 +557,47 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return cursor; } +static PyObject * +connection_get_row_factory(PyObject *op, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + return Py_NewRef(self->row_factory); +} + +static int +connection_set_row_factory(PyObject *op, PyObject *value, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, + "cannot delete row_factory attribute"); + return -1; + } + Py_XSETREF(self->row_factory, Py_NewRef(value)); + return 0; +} + +static PyObject * +connection_get_text_factory(PyObject *op, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + return Py_NewRef(self->text_factory); +} + +static int +connection_set_text_factory(PyObject *op, PyObject *value, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, + "cannot delete text_factory attribute"); + return -1; + } + Py_XSETREF(self->text_factory, Py_NewRef(value)); + return 0; +} + + /*[clinic input] _sqlite3.Connection.blobopen as blobopen @@ -1561,14 +1602,13 @@ pysqlite_connection_set_authorizer_impl(pysqlite_Connection *self, } /*[clinic input] -@permit_long_docstring_body _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_handler cls: defining_class progress_handler as callable: object A callable that takes no arguments. - If the callable returns non-zero, the current query is terminated, - and an exception is raised. + If the callable returns non-zero, the current query is + terminated, and an exception is raised. / n: int The number of SQLite virtual machine instructions that are @@ -1576,14 +1616,15 @@ _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_han Set progress handler callback. -If 'progress_handler' is None or 'n' is 0, the progress handler is disabled. +If 'progress_handler' is None or 'n' is 0, the progress handler is +disabled. [clinic start generated code]*/ static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n) -/*[clinic end generated code: output=0739957fd8034a50 input=3ecce6c915922ad4]*/ +/*[clinic end generated code: output=0739957fd8034a50 input=fd0d5abb004f370f]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1606,6 +1647,7 @@ pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, } /*[clinic input] +@permit_long_summary _sqlite3.Connection.set_trace_callback as pysqlite_connection_set_trace_callback cls: defining_class @@ -1619,7 +1661,7 @@ static PyObject * pysqlite_connection_set_trace_callback_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable) -/*[clinic end generated code: output=d91048c03bfcee05 input=f4f59bf2f87f2026]*/ +/*[clinic end generated code: output=d91048c03bfcee05 input=edfd7d890200a9cb]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2212,7 +2254,6 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self, #ifdef PY_SQLITE_HAVE_SERIALIZE /*[clinic input] -@permit_long_docstring_body _sqlite3.Connection.serialize as serialize * @@ -2221,15 +2262,15 @@ _sqlite3.Connection.serialize as serialize Serialize a database into a byte string. -For an ordinary on-disk database file, the serialization is just a copy of the -disk file. For an in-memory database or a "temp" database, the serialization is -the same sequence of bytes which would be written to disk if that database -were backed up to disk. +For an ordinary on-disk database file, the serialization is just +a copy of the disk file. For an in-memory database or a "temp" +database, the serialization is the same sequence of bytes which +would be written to disk if that database were backed up to disk. [clinic start generated code]*/ static PyObject * serialize_impl(pysqlite_Connection *self, const char *name) -/*[clinic end generated code: output=97342b0e55239dd3 input=963e617cdf75c747]*/ +/*[clinic end generated code: output=97342b0e55239dd3 input=7e48654e8e082fa8]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2263,7 +2304,6 @@ serialize_impl(pysqlite_Connection *self, const char *name) } /*[clinic input] -@permit_long_docstring_body _sqlite3.Connection.deserialize as deserialize data: Py_buffer(accept={buffer, str}) @@ -2275,18 +2315,19 @@ _sqlite3.Connection.deserialize as deserialize Load a serialized database. -The deserialize interface causes the database connection to disconnect from the -target database, and then reopen it as an in-memory database based on the given -serialized data. +The deserialize interface causes the database connection to +disconnect from the target database, and then reopen it as +an in-memory database based on the given serialized data. -The deserialize interface will fail with SQLITE_BUSY if the database is -currently in a read transaction or is involved in a backup operation. +The deserialize interface will fail with SQLITE_BUSY if the database +is currently in a read transaction or is involved in a backup +operation. [clinic start generated code]*/ static PyObject * deserialize_impl(pysqlite_Connection *self, Py_buffer *data, const char *name) -/*[clinic end generated code: output=e394c798b98bad89 input=037e94599aaa5b5c]*/ +/*[clinic end generated code: output=e394c798b98bad89 input=5d20e028d98c0686]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2359,13 +2400,14 @@ _sqlite3.Connection.__exit__ as pysqlite_connection_exit Called when the connection is used as a context manager. -If there was any exception, a rollback takes place; otherwise we commit. +If there was any exception, a rollback takes place; otherwise we +commit. [clinic start generated code]*/ static PyObject * pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=0705200e9321202a input=bd66f1532c9c54a7]*/ +/*[clinic end generated code: output=0705200e9321202a input=8fdb0392ee6f3466]*/ { int commit = 0; PyObject* result; @@ -2400,26 +2442,25 @@ pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, } /*[clinic input] -@permit_long_docstring_body _sqlite3.Connection.setlimit as setlimit category: int The limit category to be set. limit: int - The new limit. If the new limit is a negative number, the limit is - unchanged. + The new limit. If the new limit is a negative number, the limit + is unchanged. / Set connection run-time limits. -Attempts to increase a limit above its hard upper bound are silently truncated -to the hard upper bound. Regardless of whether or not the limit was changed, -the prior value of the limit is returned. +Attempts to increase a limit above its hard upper bound are silently +truncated to the hard upper bound. Regardless of whether or not the +limit was changed, the prior value of the limit is returned. [clinic start generated code]*/ static PyObject * setlimit_impl(pysqlite_Connection *self, int category, int limit) -/*[clinic end generated code: output=0d208213f8d68ccd input=bf06e06a21eb37e2]*/ +/*[clinic end generated code: output=0d208213f8d68ccd input=5c2e430091206677]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2620,6 +2661,10 @@ static PyGetSetDef connection_getset[] = { {"in_transaction", pysqlite_connection_get_in_transaction, NULL}, {"autocommit", get_autocommit, set_autocommit}, {"__text_signature__", get_sig, NULL}, + {"row_factory", connection_get_row_factory, + connection_set_row_factory}, + {"text_factory", connection_get_text_factory, + connection_set_text_factory}, {NULL} }; @@ -2667,8 +2712,6 @@ static struct PyMemberDef connection_members[] = {"InternalError", _Py_T_OBJECT, offsetof(pysqlite_Connection, InternalError), Py_READONLY}, {"ProgrammingError", _Py_T_OBJECT, offsetof(pysqlite_Connection, ProgrammingError), Py_READONLY}, {"NotSupportedError", _Py_T_OBJECT, offsetof(pysqlite_Connection, NotSupportedError), Py_READONLY}, - {"row_factory", _Py_T_OBJECT, offsetof(pysqlite_Connection, row_factory)}, - {"text_factory", _Py_T_OBJECT, offsetof(pysqlite_Connection, text_factory)}, {NULL} }; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 044eb6e5f1fb662..9964532a7f401c1 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -42,6 +42,7 @@ static const char copyright[] = #include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION #include "pycore_dict.h" // _PyDict_Next() #include "pycore_long.h" // _PyLong_GetZero() +#include "pycore_list.h" // _PyList_AppendTakeRef() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_tuple.h" // _PyTuple_FromPairSteal #include "pycore_unicodeobject.h" // _PyUnicode_Copy @@ -548,10 +549,16 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, memset(state, 0, sizeof(SRE_STATE)); - state->mark = PyMem_New(const void *, pattern->groups * 2); - if (!state->mark) { - PyErr_NoMemory(); - goto err; + /* Patterns with no capturing groups never emit MARK opcodes and never + read state->mark (group 0's span comes from state->start/ptr), so skip + the allocation entirely -- state->mark stays NULL, which both the err + path and state_fini already free safely. */ + if (pattern->groups) { + state->mark = PyMem_New(const void *, pattern->groups * 2); + if (!state->mark) { + PyErr_NoMemory(); + goto err; + } } state->lastmark = -1; state->lastindex = -1; @@ -980,8 +987,7 @@ _sre_SRE_Pattern_findall_impl(PatternObject *self, PyObject *string, break; } - status = PyList_Append(list, item); - Py_DECREF(item); + status = _PyList_AppendTakeRef((PyListObject *)list, item); if (status < 0) goto error; @@ -1327,8 +1333,7 @@ pattern_subx(_sremodulestate* module_state, string, i, b); if (!item) goto error; - status = PyList_Append(list, item); - Py_DECREF(item); + status = _PyList_AppendTakeRef((PyListObject *)list, item); if (status < 0) goto error; @@ -1357,8 +1362,7 @@ pattern_subx(_sremodulestate* module_state, /* add to list */ if (item != Py_None) { - status = PyList_Append(list, item); - Py_DECREF(item); + status = _PyList_AppendTakeRef((PyListObject *)list, item); if (status < 0) goto error; } @@ -1375,8 +1379,7 @@ pattern_subx(_sremodulestate* module_state, string, i, state.endpos); if (!item) goto error; - status = PyList_Append(list, item); - Py_DECREF(item); + status = _PyList_AppendTakeRef((PyListObject *)list, item); if (status < 0) goto error; } @@ -1839,6 +1842,34 @@ _sre_template_impl(PyObject *module, PyObject *pattern, PyObject *template) } while (0) #define GET_SKIP GET_SKIP_ADJ(0) +static int +_validate_category(SRE_CODE arg) +{ + switch (arg) { + case SRE_CATEGORY_DIGIT: + case SRE_CATEGORY_NOT_DIGIT: + case SRE_CATEGORY_SPACE: + case SRE_CATEGORY_NOT_SPACE: + case SRE_CATEGORY_WORD: + case SRE_CATEGORY_NOT_WORD: + case SRE_CATEGORY_LINEBREAK: + case SRE_CATEGORY_NOT_LINEBREAK: + case SRE_CATEGORY_LOC_WORD: + case SRE_CATEGORY_LOC_NOT_WORD: + case SRE_CATEGORY_UNI_DIGIT: + case SRE_CATEGORY_UNI_NOT_DIGIT: + case SRE_CATEGORY_UNI_SPACE: + case SRE_CATEGORY_UNI_NOT_SPACE: + case SRE_CATEGORY_UNI_WORD: + case SRE_CATEGORY_UNI_NOT_WORD: + case SRE_CATEGORY_UNI_LINEBREAK: + case SRE_CATEGORY_UNI_NOT_LINEBREAK: + return 1; + default: + return 0; + } +} + static int _validate_charset(SRE_CODE *code, SRE_CODE *end) { @@ -1891,27 +1922,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) case SRE_OP_CATEGORY: GET_ARG; - switch (arg) { - case SRE_CATEGORY_DIGIT: - case SRE_CATEGORY_NOT_DIGIT: - case SRE_CATEGORY_SPACE: - case SRE_CATEGORY_NOT_SPACE: - case SRE_CATEGORY_WORD: - case SRE_CATEGORY_NOT_WORD: - case SRE_CATEGORY_LINEBREAK: - case SRE_CATEGORY_NOT_LINEBREAK: - case SRE_CATEGORY_LOC_WORD: - case SRE_CATEGORY_LOC_NOT_WORD: - case SRE_CATEGORY_UNI_DIGIT: - case SRE_CATEGORY_UNI_NOT_DIGIT: - case SRE_CATEGORY_UNI_SPACE: - case SRE_CATEGORY_UNI_NOT_SPACE: - case SRE_CATEGORY_UNI_WORD: - case SRE_CATEGORY_UNI_NOT_WORD: - case SRE_CATEGORY_UNI_LINEBREAK: - case SRE_CATEGORY_UNI_NOT_LINEBREAK: - break; - default: + if (!_validate_category(arg)) { FAIL; } break; @@ -1992,6 +2003,13 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) } break; + case SRE_OP_CATEGORY: + GET_ARG; + if (!_validate_category(arg)) { + FAIL; + } + break; + case SRE_OP_ANY: case SRE_OP_ANY_ALL: /* These have no operands */ @@ -2587,6 +2605,7 @@ _pair(Py_ssize_t i1, Py_ssize_t i2) } /*[clinic input] +@permit_long_summary _sre.SRE_Match.span group: object(c_default="NULL") = 0 @@ -2597,7 +2616,7 @@ For match object m, return the 2-tuple (m.start(group), m.end(group)). static PyObject * _sre_SRE_Match_span_impl(MatchObject *self, PyObject *group) -/*[clinic end generated code: output=f02ae40594d14fe6 input=8fa6014e982d71d4]*/ +/*[clinic end generated code: output=f02ae40594d14fe6 input=834cfe444f0f55cf]*/ { Py_ssize_t index = match_getindex(self, group); @@ -3179,7 +3198,7 @@ static PyMethodDef pattern_methods[] = { _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("Patterns are generic over the type of string they handle (str or bytes)")}, {NULL, NULL} }; @@ -3235,7 +3254,7 @@ static PyMethodDef match_methods[] = { _SRE_SRE_MATCH___COPY___METHODDEF _SRE_SRE_MATCH___DEEPCOPY___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("Matches are generic over the type of string which was matched (str or bytes)")}, {NULL, NULL} }; diff --git a/Modules/_sre/sre_lib.h b/Modules/_sre/sre_lib.h index df377905bfae0d0..6e6ae46f05a50f0 100644 --- a/Modules/_sre/sre_lib.h +++ b/Modules/_sre/sre_lib.h @@ -193,6 +193,7 @@ LOCAL(Py_ssize_t) SRE(count)(SRE_STATE* state, const SRE_CODE* pattern, Py_ssize_t maxcount) { SRE_CODE chr; + SRE_CODE arg; SRE_CHAR c; const SRE_CHAR* ptr = (const SRE_CHAR *)state->ptr; const SRE_CHAR* end = (const SRE_CHAR *)state->end; @@ -302,6 +303,13 @@ SRE(count)(SRE_STATE* state, const SRE_CODE* pattern, Py_ssize_t maxcount) ptr++; break; + case SRE_OP_CATEGORY: + arg = pattern[1]; + TRACE(("|%p|%p|COUNT CATEGORY %d\n", pattern, ptr, arg)); + while (ptr < end && sre_category(arg, *ptr)) + ptr++; + break; + default: /* repeated single character pattern */ TRACE(("|%p|%p|COUNT SUBPATTERN\n", pattern, ptr)); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 3224ca7d0f93b96..f451c0ce7364ab9 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -26,6 +26,7 @@ #define OPENSSL_NO_DEPRECATED 1 #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_long.h" // _PyLong_UnsignedLongLong_Converter() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() @@ -2357,27 +2358,26 @@ _ssl__SSLSocket_compression_impl(PySSLSocket *self) } /*[clinic input] -@permit_long_docstring_body @critical_section @getter _ssl._SSLSocket.context This changes the context associated with the SSLSocket. -This is typically used from within a callback function set by the sni_callback -on the SSLContext to change the certificate information associated with the -SSLSocket before the cryptographic exchange handshake messages. +This is typically used from within a callback function set by the +sni_callback on the SSLContext to change the certificate information +associated with the SSLSocket before the cryptographic exchange +handshake messages. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_context_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=d23e82f72f32e3d7 input=0cc8e773a079295e]*/ +/*[clinic end generated code: output=d23e82f72f32e3d7 input=b845dea1f9710ebe]*/ { return Py_NewRef(self->ctx); } /*[clinic input] -@permit_long_docstring_body @critical_section @setter _ssl._SSLSocket.context @@ -2385,7 +2385,7 @@ _ssl._SSLSocket.context static int _ssl__SSLSocket_context_set_impl(PySSLSocket *self, PyObject *value) -/*[clinic end generated code: output=6b0a6cc5cf33d9fe input=f7fc1674b660df96]*/ +/*[clinic end generated code: output=6b0a6cc5cf33d9fe input=48ece77724fd9dd4]*/ { if (PyObject_TypeCheck(value, self->ctx->state->PySSLContext_Type)) { Py_SETREF(self->ctx, (PySSLContext *)Py_NewRef(value)); @@ -2635,7 +2635,6 @@ _ssl__SSLSocket_uses_ktls_for_recv_impl(PySSLSocket *self) #ifdef BIO_get_ktls_send /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section _ssl._SSLSocket.sendfile fd: int @@ -2646,9 +2645,9 @@ _ssl._SSLSocket.sendfile Write size bytes from offset in the file descriptor fd to the SSL connection. -This method uses the zero-copy technique and returns the number of bytes -written. It should be called only when Kernel TLS is used for sending data in -the connection. +This method uses the zero-copy technique and returns the number of +bytes written. It should be called only when Kernel TLS is used for +sending data in the connection. The meaning of flags is platform dependent. [clinic start generated code]*/ @@ -2656,7 +2655,7 @@ The meaning of flags is platform dependent. static PyObject * _ssl__SSLSocket_sendfile_impl(PySSLSocket *self, int fd, Py_off_t offset, size_t size, int flags) -/*[clinic end generated code: output=0c6815b0719ca8d5 input=1f193e681bbae664]*/ +/*[clinic end generated code: output=0c6815b0719ca8d5 input=68c7fbf90c9a8a1b]*/ { Py_ssize_t retval; int sockstate; @@ -3175,22 +3174,22 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } /*[clinic input] -@permit_long_docstring_body @critical_section _ssl._SSLSocket.get_channel_binding cb_type: str = "tls-unique" Get channel binding data for current connection. -Raise ValueError if the requested `cb_type` is not supported. Return bytes -of the data or None if the data is not available (e.g. before the handshake). +Raise ValueError if the requested `cb_type` is not supported. +Return bytes of the data or None if the data is not available (e.g. +before the handshake). Only 'tls-unique' channel binding data from RFC 5929 is supported. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, const char *cb_type) -/*[clinic end generated code: output=34bac9acb6a61d31 input=26fad522435ecca1]*/ +/*[clinic end generated code: output=34bac9acb6a61d31 input=bed81ef7936535a0]*/ { char buf[PySSL_CB_MAXLEN]; size_t len; @@ -5153,12 +5152,15 @@ _servername_callback(SSL *s, int *al, void *args) PyObject *result; /* The high-level ssl.SSLSocket object */ PyObject *ssl_socket; + PyObject *sni_cb; const char *servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name); PyGILState_STATE gstate = PyGILState_Ensure(); - if (sslctx->set_sni_cb == NULL) { - /* remove race condition in this the call back while if removing the - * callback is in progress */ + Py_BEGIN_CRITICAL_SECTION(sslctx); + sni_cb = Py_XNewRef(sslctx->set_sni_cb); + Py_END_CRITICAL_SECTION(); + + if (sni_cb == NULL) { PyGILState_Release(gstate); return SSL_TLSEXT_ERR_OK; } @@ -5185,7 +5187,7 @@ _servername_callback(SSL *s, int *al, void *args) goto error; if (servername == NULL) { - result = PyObject_CallFunctionObjArgs(sslctx->set_sni_cb, ssl_socket, + result = PyObject_CallFunctionObjArgs(sni_cb, ssl_socket, Py_None, sslctx, NULL); } else { @@ -5212,7 +5214,7 @@ _servername_callback(SSL *s, int *al, void *args) } Py_DECREF(servername_bytes); result = PyObject_CallFunctionObjArgs( - sslctx->set_sni_cb, ssl_socket, servername_str, + sni_cb, ssl_socket, servername_str, sslctx, NULL); Py_DECREF(servername_str); } @@ -5222,7 +5224,7 @@ _servername_callback(SSL *s, int *al, void *args) PyErr_FormatUnraisable("Exception ignored " "in ssl servername callback " "while calling set SNI callback %R", - sslctx->set_sni_cb); + sni_cb); *al = SSL_AD_HANDSHAKE_FAILURE; ret = SSL_TLSEXT_ERR_ALERT_FATAL; } @@ -5247,11 +5249,13 @@ _servername_callback(SSL *s, int *al, void *args) Py_DECREF(result); } + Py_DECREF(sni_cb); PyGILState_Release(gstate); return ret; error: Py_XDECREF(ssl_socket); + Py_XDECREF(sni_cb); *al = SSL_AD_INTERNAL_ERROR; ret = SSL_TLSEXT_ERR_ALERT_FATAL; PyGILState_Release(gstate); @@ -5260,22 +5264,22 @@ _servername_callback(SSL *s, int *al, void *args) /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section @getter _ssl._SSLContext.sni_callback Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension. -If the argument is None then the callback is disabled. The method is called -with the SSLSocket, the server name as a string, and the SSLContext object. +If the argument is None then the callback is disabled. The method +is called with the SSLSocket, the server name as a string, and the +SSLContext object. See RFC 6066 for details of the SNI extension. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) -/*[clinic end generated code: output=961e6575cdfaf036 input=3aee06696b0874d9]*/ +/*[clinic end generated code: output=961e6575cdfaf036 input=a319bc8fc15d6fc8]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -5286,7 +5290,6 @@ _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section @setter _ssl._SSLContext.sni_callback @@ -5294,27 +5297,25 @@ _ssl._SSLContext.sni_callback static int _ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value) -/*[clinic end generated code: output=b32736c6b891f61a input=332def1d8c81d549]*/ +/*[clinic end generated code: output=b32736c6b891f61a input=402b43fb06c1139e]*/ { if (self->protocol == PY_SSL_VERSION_TLS_CLIENT) { PyErr_SetString(PyExc_ValueError, "sni_callback cannot be set on TLS_CLIENT context"); return -1; } - Py_CLEAR(self->set_sni_cb); - if (value == Py_None) { + if (!PyCallable_Check(value)) { SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); - } - else { - if (!PyCallable_Check(value)) { - SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); - PyErr_SetString(PyExc_TypeError, - "not a callable object"); + Py_CLEAR(self->set_sni_cb); + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, "not a callable object"); return -1; } - self->set_sni_cb = Py_NewRef(value); - SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); + } + else { + Py_XSETREF(self->set_sni_cb, Py_NewRef(value)); SSL_CTX_set_tlsext_servername_arg(self->ctx, self); + SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); } return 0; } @@ -5369,16 +5370,16 @@ _ssl._SSLContext.cert_store_stats Returns quantities of loaded X.509 certificates. -X.509 certificates with a CA extension and certificate revocation lists -inside the context's cert store. +X.509 certificates with a CA extension and certificate revocation +lists inside the context's cert store. -NOTE: Certificates in a capath directory aren't loaded unless they have -been used at least once. +NOTE: Certificates in a capath directory aren't loaded unless they +have been used at least once. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) -/*[clinic end generated code: output=5f356f4d9cca874d input=d13c6e3f2b48539b]*/ +/*[clinic end generated code: output=5f356f4d9cca874d input=9e5094e094b892a3]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -5421,16 +5422,16 @@ _ssl._SSLContext.get_ca_certs Returns a list of dicts with information of loaded CA certs. -If the optional argument is True, returns a DER-encoded copy of the CA -certificate. +If the optional argument is True, returns a DER-encoded copy of the +CA certificate. -NOTE: Certificates in a capath directory aren't loaded unless they have -been used at least once. +NOTE: Certificates in a capath directory aren't loaded unless they +have been used at least once. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) -/*[clinic end generated code: output=0d58f148f37e2938 input=eb0592909c9ad6e7]*/ +/*[clinic end generated code: output=0d58f148f37e2938 input=9f71af5aa4e67076]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -6319,13 +6320,13 @@ _ssl.RAND_status Returns True if the OpenSSL PRNG has been seeded with enough data and False if not. -It is necessary to seed the PRNG with RAND_add() on some platforms before -using the ssl() function. +It is necessary to seed the PRNG with RAND_add() on some platforms +before using the ssl() function. [clinic start generated code]*/ static PyObject * _ssl_RAND_status_impl(PyObject *module) -/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=aba24a3f3af3b184]*/ +/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=52b061f4a24ff3a1]*/ { return PyBool_FromLong(RAND_status()); } @@ -6621,16 +6622,16 @@ _ssl.enum_certificates Retrieve certificates from Windows' cert store. -store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide -more cert storages, too. The function returns a list of (bytes, -encoding_type, trust) tuples. The encoding_type flag can be interpreted -with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either -a set of OIDs or the boolean True. +store_name may be one of 'CA', 'ROOT' or 'MY'. The system may +provide more cert storages, too. The function returns a list of +(bytes, encoding_type, trust) tuples. The encoding_type flag can be +interpreted with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The +trust setting is either a set of OIDs or the boolean True. [clinic start generated code]*/ static PyObject * _ssl_enum_certificates_impl(PyObject *module, const char *store_name) -/*[clinic end generated code: output=5134dc8bb3a3c893 input=263c22e6c6988cf3]*/ +/*[clinic end generated code: output=5134dc8bb3a3c893 input=ef81b4bd1b7ab8e9]*/ { HCERTSTORE hCollectionStore = NULL; PCCERT_CONTEXT pCertCtx = NULL; diff --git a/Modules/_struct.c b/Modules/_struct.c index 3a970d99bb3d6d8..8c611a708d02a9d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2721,6 +2721,7 @@ pack_into_impl(PyObject *module, PyStructObject *s_object, Py_buffer *buffer, } /*[clinic input] +@permit_long_summary unpack format as s_object: cache_struct @@ -2735,12 +2736,13 @@ for more on format strings. static PyObject * unpack_impl(PyObject *module, PyStructObject *s_object, Py_buffer *buffer) -/*[clinic end generated code: output=48ddd4d88eca8551 input=7df28c5d0b5b6f4e]*/ +/*[clinic end generated code: output=48ddd4d88eca8551 input=53a60a65830bd1e1]*/ { return Struct_unpack_impl(s_object, buffer); } /*[clinic input] +@permit_long_summary unpack_from format as s_object: cache_struct @@ -2757,7 +2759,7 @@ help(struct) for more on format strings. static PyObject * unpack_from_impl(PyObject *module, PyStructObject *s_object, Py_buffer *buffer, Py_ssize_t offset) -/*[clinic end generated code: output=1042631674c6e0d3 input=599262b23559f6c5]*/ +/*[clinic end generated code: output=1042631674c6e0d3 input=3e46619756fb0293]*/ { return Struct_unpack_from_impl(s_object, buffer, offset); } diff --git a/Modules/_testcapi/import.c b/Modules/_testcapi/import.c index ebb1032fdd1c320..11d0e6acaebe1f2 100644 --- a/Modules/_testcapi/import.c +++ b/Modules/_testcapi/import.c @@ -1,6 +1,27 @@ #include "parts.h" #include "util.h" +static PyObject * +pyimport_lazyimportwithoutframe(PyObject *self, PyObject *name) +{ + PyObject *lazy_import = PyImport_ImportModuleAttrString("builtins", + "__lazy_import__"); + if (lazy_import == NULL) { + return NULL; + } + + // Simulate being called with no running Python frame (e.g. from a freshly + // attached C thread), so that PyEval_GetGlobals() returns NULL. + PyThreadState *tstate = PyThreadState_Get(); + struct _PyInterpreterFrame *saved = tstate->current_frame; + tstate->current_frame = NULL; + PyObject *res = PyObject_CallOneArg(lazy_import, name); + tstate->current_frame = saved; + + Py_DECREF(lazy_import); + return res; +} + // Test PyImport_ImportModuleAttr() static PyObject * pyimport_importmoduleattr(PyObject *self, PyObject *args) @@ -41,8 +62,6 @@ pyimport_setlazyimportsmode(PyObject *self, PyObject *args) PyImport_SetLazyImportsMode(PyImport_LAZY_NORMAL); } else if (strcmp(PyUnicode_AsUTF8(mode), "all") == 0) { PyImport_SetLazyImportsMode(PyImport_LAZY_ALL); - } else if (strcmp(PyUnicode_AsUTF8(mode), "none") == 0) { - PyImport_SetLazyImportsMode(PyImport_LAZY_NONE); } else { PyErr_SetString(PyExc_ValueError, "invalid mode"); return NULL; @@ -59,8 +78,6 @@ pyimport_getlazyimportsmode(PyObject *self, PyObject *args) return PyUnicode_FromString("normal"); case PyImport_LAZY_ALL: return PyUnicode_FromString("all"); - case PyImport_LAZY_NONE: - return PyUnicode_FromString("none"); default: PyErr_SetString(PyExc_ValueError, "unknown mode"); return NULL; @@ -99,6 +116,7 @@ static PyMethodDef test_methods[] = { {"PyImport_GetLazyImportsMode", pyimport_getlazyimportsmode, METH_NOARGS}, {"PyImport_SetLazyImportsFilter", pyimport_setlazyimportsfilter, METH_VARARGS}, {"PyImport_GetLazyImportsFilter", pyimport_getlazyimportsfilter, METH_NOARGS}, + {"lazy_import_without_frame", pyimport_lazyimportwithoutframe, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index b4896f984510bd6..28e89275d61705e 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -2,6 +2,28 @@ #include <stddef.h> +#if defined(__APPLE__) +# include <TargetConditionals.h> + // Older macOS SDKs do not define TARGET_OS_OSX +# if !defined(TARGET_OS_OSX) +# define TARGET_OS_OSX 1 +# endif +# if TARGET_OS_OSX +# include <errno.h> // errno, ESRCH +# include <libproc.h> // proc_pidinfo(), PROC_PIDTASKINFO +# include <sys/proc_info.h> // struct proc_taskinfo +# endif +#endif + +#ifdef __FreeBSD__ +# include <fcntl.h> // O_RDONLY +# include <kvm.h> // kvm_openfiles() +# include <limits.h> // _POSIX2_LINE_MAX +# include <sys/sysctl.h> // KERN_PROC_PID +# include <sys/user.h> // kinfo_proc definition +# include <unistd.h> // sysconf() +#endif + typedef struct { PyMemAllocatorEx alloc; @@ -323,6 +345,53 @@ test_setallocators(PyMemAllocatorDomain domain) goto fail; } + /* realloc(NULL, size) should behave like malloc(size) */ + size_t size3 = 100; + void *ptr3; + switch(domain) { + case PYMEM_DOMAIN_RAW: + ptr3 = PyMem_RawRealloc(NULL, size3); + break; + case PYMEM_DOMAIN_MEM: + ptr3 = PyMem_Realloc(NULL, size3); + break; + case PYMEM_DOMAIN_OBJ: + ptr3 = PyObject_Realloc(NULL, size3); + break; + default: + ptr3 = NULL; + break; + } + + CHECK_CTX("realloc(NULL, size)"); + if (ptr3 == NULL) { + error_msg = "realloc(NULL, size) failed"; + goto fail; + } + if (hook.realloc_ptr != NULL || hook.realloc_new_size != size3) { + error_msg = "realloc(NULL, size) invalid parameters"; + goto fail; + } + + hook.free_ptr = NULL; + switch(domain) { + case PYMEM_DOMAIN_RAW: + PyMem_RawFree(ptr3); + break; + case PYMEM_DOMAIN_MEM: + PyMem_Free(ptr3); + break; + case PYMEM_DOMAIN_OBJ: + PyObject_Free(ptr3); + break; + } + + CHECK_CTX("realloc(NULL, size) free"); + if (hook.free_ptr != ptr3) { + error_msg = "unexpected pointer passed to free"; + goto fail; + } + res = Py_NewRef(Py_None); goto finally; @@ -684,6 +753,74 @@ tracemalloc_track_race(PyObject *self, PyObject *args) } +#if TARGET_OS_OSX || defined(__FreeBSD__) +// Return RSS only. Per-process swap usage isn't readily available +static PyObject* +get_process_memory_usage(PyObject *self, PyObject *args) +{ + int pid; + if (!PyArg_ParseTuple(args, "i", &pid)) { + return NULL; + } + +#if TARGET_OS_OSX + // macOS: proc_pidinfo(PROC_PIDTASKINFO).pti_resident_size + struct proc_taskinfo pti; + int ret = proc_pidinfo(pid, PROC_PIDTASKINFO, 0, &pti, sizeof(pti)); + if (ret <= 0) { + if (errno == 0) { + // proc_pidinfo() can return 0 without setting errno when the + // process does not exist. + errno = ESRCH; + } + return PyErr_SetFromErrno(PyExc_OSError); + } + + return PyLong_FromUnsignedLongLong(pti.pti_resident_size); +#else + // FreeBSD: kvm_getprocs(KERN_PROC_PID) and ki_rssize * page_size + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + return PyErr_SetFromErrno(PyExc_OSError); + } + + // Using /dev/null for vmcore avoids needing dump file. + // NULL for kernel file uses running kernel. + char errbuf[_POSIX2_LINE_MAX]; + kvm_t *kd = kvm_openfiles(NULL, "/dev/null", NULL, O_RDONLY, errbuf); + if (kd == NULL) { + return PyErr_SetFromErrno(PyExc_OSError); + } + + // KERN_PROC_PID filters for the specific process ID. + int n_procs; + struct kinfo_proc *kp = kvm_getprocs(kd, KERN_PROC_PID, pid, &n_procs); + if (kp == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (n_procs <= 0) { + // Process with PID not found + errno = ESRCH; + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + assert(n_procs == 1); + + // ki_rssize is in pages. Convert to bytes. + size_t rss = (size_t)kp[0].ki_rssize * page_size; + kvm_close(kd); + + return PyLong_FromSize_t(rss); + +error: + kvm_close(kd); + return NULL; +#endif +} +#endif + + static PyMethodDef test_methods[] = { {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, @@ -698,6 +835,9 @@ static PyMethodDef test_methods[] = { {"test_pymem_setrawallocators", test_pymem_setrawallocators, METH_NOARGS}, {"test_pyobject_new", test_pyobject_new, METH_NOARGS}, {"test_pyobject_setallocators", test_pyobject_setallocators, METH_NOARGS}, +#if TARGET_OS_OSX || defined(__FreeBSD__) + {"get_process_memory_usage", get_process_memory_usage, METH_VARARGS}, +#endif // Tracemalloc tests {"tracemalloc_track", tracemalloc_track, METH_VARARGS}, diff --git a/Modules/_testcapi/module.c b/Modules/_testcapi/module.c index 29eda7e008c6cb3..b0b4a4f4c39d8f9 100644 --- a/Modules/_testcapi/module.c +++ b/Modules/_testcapi/module.c @@ -13,8 +13,8 @@ PyABIInfo_VAR(abi_info); static PyObject * module_from_slots_empty(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {0}, + PySlot slots[] = { + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -22,9 +22,9 @@ module_from_slots_empty(PyObject *self, PyObject *spec) static PyObject * module_from_slots_minimal(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -38,12 +38,12 @@ module_from_slots_null(PyObject *self, PyObject *spec) static PyObject * module_from_slots_name(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "currently ignored..."}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_name, "currently ignored..."), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -51,12 +51,12 @@ module_from_slots_name(PyObject *self, PyObject *spec) static PyObject * module_from_slots_doc(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_doc, "the docstring"}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_doc, "the docstring"), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -64,12 +64,12 @@ module_from_slots_doc(PyObject *self, PyObject *spec) static PyObject * module_from_slots_size(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_state_size, (void*)123}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_SIZE(Py_mod_state_size, (void*)123), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); if (!mod) { @@ -92,12 +92,12 @@ static PyMethodDef a_methoddef_array[] = { static PyObject * module_from_slots_methods(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_methods, a_methoddef_array}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_methods, a_methoddef_array), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -111,14 +111,14 @@ static void noop_free(void *self) { } static PyObject * module_from_slots_gc(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_state_traverse, noop_traverse}, - {Py_mod_state_clear, noop_clear}, - {Py_mod_state_free, noop_free}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_FUNC(Py_mod_state_traverse, noop_traverse), + PySlot_FUNC(Py_mod_state_clear, noop_clear), + PySlot_FUNC(Py_mod_state_free, noop_free), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); if (!mod) { @@ -144,12 +144,12 @@ static const char test_token; static PyObject * module_from_slots_token(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_token, (void*)&test_token}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_token, (void*)&test_token), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); if (!mod) { @@ -175,12 +175,12 @@ simple_exec(PyObject *module) static PyObject * module_from_slots_exec(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_exec, simple_exec}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_FUNC(Py_mod_exec, simple_exec), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; PyObject *mod = PyModule_FromSlotsAndSpec(slots, spec); if (!mod) { @@ -209,12 +209,12 @@ create_attr_from_spec(PyObject *spec, PyModuleDef *def) static PyObject * module_from_slots_create(PyObject *self, PyObject *spec) { - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_create, create_attr_from_spec}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + const PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_FUNC(Py_mod_create, create_attr_from_spec), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -241,13 +241,13 @@ module_from_slots_repeat_slot(PyObject *self, PyObject *spec) if (slot_id < 0) { return NULL; } - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {slot_id, "anything"}, - {slot_id, "anything else"}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + const PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_PTR_STATIC(slot_id, "anything"), + PySlot_PTR_STATIC(slot_id, "anything_else"), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -259,12 +259,12 @@ module_from_slots_null_slot(PyObject *self, PyObject *spec) if (slot_id < 0) { return NULL; } - PyModuleDef_Slot slots[] = { - {slot_id, NULL}, - {Py_mod_abi, &abi_info}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + const PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_PTR_STATIC(slot_id, NULL), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -347,12 +347,12 @@ module_from_bad_abiinfo(PyObject *self, PyObject *spec) 1, 0, .abi_version=0x02080000, }; - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_abi, &bad_abi_info}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_abi, &bad_abi_info), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -365,14 +365,14 @@ module_from_multiple_abiinfo(PyObject *self, PyObject *spec) .flags=PyABIInfo_STABLE | PyABIInfo_FREETHREADING_AGNOSTIC, .abi_version=0x03040000, }; - PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_abi, &abi_info}, - {Py_mod_abi, &extra_abi_info}, - {Py_mod_abi, &extra_abi_info}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_abi, &extra_abi_info), + PySlot_DATA(Py_mod_abi, &extra_abi_info), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return PyModule_FromSlotsAndSpec(slots, spec); } @@ -466,6 +466,53 @@ pymodule_get_state_size(PyObject *self, PyObject *module) return PyLong_FromSsize_t(size); } +static PyObject * +module_from_null_def_slot(PyObject* Py_UNUSED(module), PyObject *args) +{ + long slot_number; + PyObject *spec; + if (!PyArg_ParseTuple(args, "lO", &slot_number, &spec)) { + return NULL; + } + static PyModuleDef_Slot slots[] = { + {0, NULL}, + {0}, + }; + static PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "mymod", + .m_slots = slots, + }; + // hack: def is supposed to be constant. + // Don't do this at home; use PyModule_FromSlotsAndSpec throwaway + // definitions! + slots[0].slot = slot_number; + return PyModule_FromDefAndSpec(&def, spec); +} + +static PyObject * +module_from_def_nonstatic_nested(PyObject* Py_UNUSED(module), PyObject *spec) +{ + static PyModuleDef_Slot subsubslots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0}, + }; + static PySlot subslots[] = { + PySlot_DATA(Py_mod_slots, subsubslots), + PySlot_END, + }; + static PyModuleDef_Slot slots[] = { + {Py_slot_subslots, subslots}, + {0}, + }; + static PyModuleDef def = { + PyModuleDef_HEAD_INIT, + .m_name = "mymod", + .m_slots = slots, + }; + return PyModule_FromDefAndSpec(&def, spec); +} + static PyMethodDef test_methods[] = { {"module_from_slots_empty", module_from_slots_empty, METH_O}, {"module_from_slots_minimal", module_from_slots_minimal, METH_O}, @@ -489,6 +536,8 @@ static PyMethodDef test_methods[] = { {"pymodule_get_def", pymodule_get_def, METH_O}, {"pymodule_get_state_size", pymodule_get_state_size, METH_O}, {"pymodule_exec", pymodule_exec, METH_O}, + {"module_from_null_def_slot", module_from_null_def_slot, METH_VARARGS}, + {"module_from_def_nonstatic_nested", module_from_def_nonstatic_nested, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 6e5c8dcbb725fa5..09a548fd2e24489 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -560,10 +560,11 @@ pysentinel_new(PyObject *self, PyObject *args) { const char *name; const char *module_name = NULL; - if (!PyArg_ParseTuple(args, "s|s", &name, &module_name)) { + const char *repr = NULL; + if (!PyArg_ParseTuple(args, "s|ss", &name, &module_name, &repr)) { return NULL; } - return PySentinel_New(name, module_name); + return PySentinel_New(name, module_name, repr); } static PyObject * @@ -572,6 +573,12 @@ pysentinel_check(PyObject *self, PyObject *obj) return PyBool_FromLong(PySentinel_Check(obj)); } +static PyObject * +pysentinel_checkexact(PyObject *self, PyObject *obj) +{ + return PyBool_FromLong(PySentinel_CheckExact(obj)); +} + static PyMethodDef test_methods[] = { {"call_pyobject_print", call_pyobject_print, METH_VARARGS}, @@ -604,6 +611,7 @@ static PyMethodDef test_methods[] = { {"pyobject_dump", pyobject_dump, METH_VARARGS}, {"pysentinel_new", pysentinel_new, METH_VARARGS}, {"pysentinel_check", pysentinel_check, METH_O}, + {"pysentinel_checkexact", pysentinel_checkexact, METH_O}, {NULL}, }; diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index a7feca5bd960705..98b5dd47accde35 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -67,5 +67,6 @@ int _PyTestCapi_Init_Frame(PyObject *mod); int _PyTestCapi_Init_Type(PyObject *mod); int _PyTestCapi_Init_Function(PyObject *mod); int _PyTestCapi_Init_Module(PyObject *mod); +int _PyTestCapi_Init_Weakref(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapi/type.c b/Modules/_testcapi/type.c index 9bef58d1f836687..f566efa0ca15ae6 100644 --- a/Modules/_testcapi/type.c +++ b/Modules/_testcapi/type.c @@ -110,9 +110,9 @@ test_get_statictype_slots(PyObject *self, PyObject *Py_UNUSED(ignored)) return NULL; } - void *over_value = PyType_GetSlot(&PyLong_Type, Py_bf_releasebuffer + 1); + void *over_value = PyType_GetSlot(&PyLong_Type, Py_mod_name + 1); if (over_value != NULL) { - PyErr_SetString(PyExc_AssertionError, "mismatch: max+1 of long"); + PyErr_SetString(PyExc_AssertionError, "mismatch: mod_name of long"); return NULL; } diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 5a756a87c15fe92..71cdc54009017a7 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -9,6 +9,7 @@ #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_interp_structs.h" // CODE_MAX_WATCHERS #include "pycore_context.h" // CONTEXT_MAX_WATCHERS +#include "pycore_lock.h" // _PyOnceFlag /*[clinic input] module _testcapi @@ -18,6 +19,14 @@ module _testcapi // Test dict watching static PyObject *g_dict_watch_events = NULL; static int g_dict_watchers_installed = 0; +static _PyOnceFlag g_dict_watch_once = {0}; + +static int +_init_dict_watch_events(void *arg) +{ + g_dict_watch_events = PyList_New(0); + return g_dict_watch_events ? 0 : -1; +} static int dict_watch_callback(PyDict_WatchEvent event, @@ -106,13 +115,10 @@ add_dict_watcher(PyObject *self, PyObject *kind) if (watcher_id < 0) { return NULL; } - if (!g_dict_watchers_installed) { - assert(!g_dict_watch_events); - if (!(g_dict_watch_events = PyList_New(0))) { - return NULL; - } + if (_PyOnceFlag_CallOnce(&g_dict_watch_once, _init_dict_watch_events, NULL) < 0) { + return NULL; } - g_dict_watchers_installed++; + _Py_atomic_add_int(&g_dict_watchers_installed, 1); return PyLong_FromLong(watcher_id); } @@ -122,10 +128,8 @@ clear_dict_watcher(PyObject *self, PyObject *watcher_id) if (PyDict_ClearWatcher(PyLong_AsLong(watcher_id))) { return NULL; } - g_dict_watchers_installed--; - if (!g_dict_watchers_installed) { - assert(g_dict_watch_events); - Py_CLEAR(g_dict_watch_events); + if (_Py_atomic_add_int(&g_dict_watchers_installed, -1) == 1) { + PyList_Clear(g_dict_watch_events); } Py_RETURN_NONE; } @@ -164,7 +168,7 @@ _testcapi_unwatch_dict_impl(PyObject *module, int watcher_id, PyObject *dict) static PyObject * get_dict_watcher_events(PyObject *self, PyObject *Py_UNUSED(args)) { - if (!g_dict_watch_events) { + if (_Py_atomic_load_int(&g_dict_watchers_installed) <= 0) { PyErr_SetString(PyExc_RuntimeError, "no watchers active"); return NULL; } @@ -212,13 +216,32 @@ type_modified_callback_error(PyTypeObject *type) return -1; } +static int +type_modified_callback_name(PyTypeObject *type) +{ + assert(PyList_Check(g_type_modified_events)); + PyObject *name = PyUnicode_FromString(type->tp_name); + if (name == NULL) { + return -1; + } + if (PyList_Append(g_type_modified_events, name) < 0) { + Py_DECREF(name); + return -1; + } + Py_DECREF(name); + return 0; +} + static PyObject * add_type_watcher(PyObject *self, PyObject *kind) { int watcher_id; assert(PyLong_Check(kind)); long kind_l = PyLong_AsLong(kind); - if (kind_l == 2) { + if (kind_l == 3) { + watcher_id = PyType_AddWatcher(type_modified_callback_name); + } + else if (kind_l == 2) { watcher_id = PyType_AddWatcher(type_modified_callback_wrap); } else if (kind_l == 1) { diff --git a/Modules/_testcapi/weakref.c b/Modules/_testcapi/weakref.c new file mode 100644 index 000000000000000..7c3ad8565991b7e --- /dev/null +++ b/Modules/_testcapi/weakref.c @@ -0,0 +1,46 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +pyweakref_getref(PyObject *module, PyObject *ref) +{ + NULLABLE(ref); + PyObject *obj = UNINITIALIZED_PTR; + int rc = PyWeakref_GetRef(ref, &obj); + if (rc == -1 && PyErr_Occurred()) { + assert(obj == NULL); + return NULL; + } + if (obj == NULL) { + return Py_BuildValue("i", rc); + } + else { + assert(obj != UNINITIALIZED_PTR); + return Py_BuildValue("iN", rc, obj); + } +} + +static PyObject * +pyweakref_isdead(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + int rc = PyWeakref_IsDead(obj); + if (rc == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(rc); +} + + +static PyMethodDef test_methods[] = { + {"pyweakref_getref", pyweakref_getref, METH_O}, + {"pyweakref_isdead", pyweakref_isdead, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Weakref(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3ebe4ceea6a72e7..799390c27053287 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -22,6 +22,9 @@ #ifdef HAVE_SYS_WAIT_H # include <sys/wait.h> // W_STOPCODE #endif +#ifdef HAVE_SYS_SYSCTL_H +# include <sys/sysctl.h> // sysctlbyname() +#endif #ifdef bool # error "The public headers should not include <stdbool.h>, see gh-48924" @@ -2606,6 +2609,334 @@ create_managed_weakref_nogc_type(PyObject *self, PyObject *Py_UNUSED(args)) return PyType_FromSpec(&ManagedWeakrefNoGC_spec); } +static void +test_interp_guards_common(void) +{ + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + assert(guard != NULL); + + PyInterpreterGuard *guard_2 = PyInterpreterGuard_FromCurrent(); + assert(guard_2 != NULL); + + // We can close the guards in any order + PyInterpreterGuard_Close(guard_2); + PyInterpreterGuard_Close(guard); +} + +static PyObject * +test_interpreter_guards(PyObject *self, PyObject *unused) +{ + // Test the main interpreter + test_interp_guards_common(); + + // Test a (legacy) subinterpreter + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + PyThreadState *interp_tstate = Py_NewInterpreter(); + // Note: For these tests, we don't bother adding error paths, because + // there's no realistic case where interpreter creation would fail here. + assert(interp_tstate != NULL); + test_interp_guards_common(); + Py_EndInterpreter(interp_tstate); + + // Test an isolated subinterpreter + PyInterpreterConfig config = { + .gil = PyInterpreterConfig_OWN_GIL, + .check_multi_interp_extensions = 1 + }; + + PyThreadState *isolated_interp_tstate; + PyStatus status = Py_NewInterpreterFromConfig(&isolated_interp_tstate, &config); + assert(!PyStatus_Exception(status)); + + test_interp_guards_common(); + Py_EndInterpreter(isolated_interp_tstate); + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_thread_state_ensure_nested(PyObject *self, PyObject *unused) +{ + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + assert(guard != NULL); + + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + assert(PyGILState_GetThisThreadState() == save_tstate); + PyThreadStateToken *tokens[10]; + + for (int i = 0; i < 10; ++i) { + // Test reactivation of the detached tstate. + tokens[i] = PyThreadState_Ensure(guard); + assert(tokens[i] != NULL); + + // No new thread state should've been created. + assert(PyThreadState_Get() == save_tstate); + PyThreadState_Release(tokens[i]); + } + + assert(PyThreadState_GetUnchecked() == NULL); + + // Similarly, test ensuring with deep nesting and *then* releasing. + // If the (detached) gilstate matches the interpreter, then it shouldn't + // create a new thread state. + for (int i = 0; i < 10; ++i) { + tokens[i] = PyThreadState_Ensure(guard); + assert(tokens[i] != NULL); + assert(PyThreadState_Get() == save_tstate); + } + + for (int i = 9; i >= 0; --i) { + assert(PyThreadState_Get() == save_tstate); + PyThreadState_Release(tokens[i]); + } + + assert(PyThreadState_GetUnchecked() == NULL); + PyInterpreterGuard_Close(guard); + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_thread_state_ensure_crossinterp(PyObject *self, PyObject *unused) +{ + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + PyThreadState *interp_tstate = Py_NewInterpreter(); + assert(interp_tstate != NULL); + + /* This should create a new thread state for the calling interpreter, *not* + reactivate the old one. In a real-world scenario, this would arise in + something like this: + + def some_func(): + import something + # This re-enters the main interpreter, but we + # shouldn't have access to prior thread-locals. + something.call_something() + + interp = interpreters.create() + interp.exec(some_func) + */ + PyThreadStateToken *token = PyThreadState_Ensure(guard); + assert(token != NULL); + + PyThreadState *ensured_tstate = PyThreadState_Get(); + assert(ensured_tstate != save_tstate); + assert(PyGILState_GetThisThreadState() == ensured_tstate); + + // Now though, we should reactivate the thread state + PyThreadStateToken *other_token = PyThreadState_Ensure(guard); + assert(other_token != NULL); + assert(PyThreadState_Get() == ensured_tstate); + + PyThreadState_Release(other_token); + + // Ensure that we're restoring the prior thread state + PyThreadState_Release(token); + assert(PyThreadState_Get() == interp_tstate); + assert(PyGILState_GetThisThreadState() == interp_tstate); + + PyThreadState_Swap(interp_tstate); + Py_EndInterpreter(interp_tstate); + + PyInterpreterGuard_Close(guard); + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_interp_view_after_shutdown(PyObject *self, PyObject *unused) +{ + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + PyThreadState *interp_tstate = Py_NewInterpreter(); + if (interp_tstate == NULL) { + PyThreadState_Swap(save_tstate); + return PyErr_NoMemory(); + } + + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + if (view == NULL) { + Py_EndInterpreter(interp_tstate); + PyThreadState_Swap(save_tstate); + return PyErr_NoMemory(); + } + + // As a sanity check, ensure that the view actually works + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + PyInterpreterGuard_Close(guard); + + // Now, destroy the interpreter and try to acquire a lock from a view. + // It should fail. + Py_EndInterpreter(interp_tstate); + guard = PyInterpreterGuard_FromView(view); + assert(guard == NULL); + + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_thread_state_ensure_view(PyObject *self, PyObject *unused) +{ + // For simplicity's sake, we assume that functions won't fail due to being + // out of memory. + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + PyThreadState *interp_tstate = Py_NewInterpreter(); + assert(interp_tstate != NULL); + assert(PyInterpreterState_Get() == PyThreadState_GetInterpreter(interp_tstate)); + + PyInterpreterView *main_view = PyInterpreterView_FromMain(); + assert(main_view != NULL); + + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + assert(view != NULL); + + Py_BEGIN_ALLOW_THREADS; + PyThreadStateToken *token = PyThreadState_EnsureFromView(view); + assert(token != NULL); + assert(PyThreadState_Get() == interp_tstate); + + // Test a nested call + PyThreadStateToken *token2 = PyThreadState_EnsureFromView(view); + assert(PyThreadState_Get() == interp_tstate); + + // We're in a new interpreter now. PyThreadState_EnsureFromView() should + // now create a new thread state. + PyThreadStateToken *main_token = PyThreadState_EnsureFromView(main_view); + assert(main_token == (PyThreadStateToken*)interp_tstate); // The old thread state + assert(PyInterpreterState_Get() == PyInterpreterState_Main()); + + // Going back to the old interpreter should create a new thread state again. + PyThreadStateToken *token3 = PyThreadState_EnsureFromView(view); + assert(PyInterpreterState_Get() == PyThreadState_GetInterpreter(interp_tstate)); + assert(PyThreadState_Get() != interp_tstate); + PyThreadState_Release(token3); + PyThreadState_Release(main_token); + + // We're back in the original interpreter. PyThreadState_EnsureFromView() should + // no longer create a new thread state. + assert(PyThreadState_Get() == interp_tstate); + PyThreadStateToken *token4 = PyThreadState_EnsureFromView(view); + assert(PyThreadState_Get() == interp_tstate); + PyThreadState_Release(token4); + PyThreadState_Release(token2); + PyThreadState_Release(token); + assert(PyThreadState_GetUnchecked() == NULL); + Py_END_ALLOW_THREADS; + + assert(PyThreadState_Get() == interp_tstate); + PyInterpreterView_Close(view); + PyInterpreterView_Close(main_view); + Py_EndInterpreter(interp_tstate); + PyThreadState_Swap(save_tstate); + + Py_RETURN_NONE; +} + +static PyObject * +test_thread_state_ensure_detachment(PyObject *self, PyObject *unused) +{ + PyThreadState *before = PyThreadState_Get(); + assert(before != NULL); + + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + assert(guard != NULL); + + PyThreadStateToken *token = PyThreadState_Ensure(guard); + assert(token != NULL); + /* Ensure took the fast path; tstate is unchanged. */ + assert(PyThreadState_Get() == before); + + PyThreadState_Release(token); + + PyThreadState *after = PyThreadState_GetUnchecked(); + assert(after != NULL); + + PyInterpreterGuard_Close(guard); + Py_RETURN_NONE; +} + +static PyObject * +test_thread_state_ensure_detached_gilstate(PyObject *self, PyObject *unused) +{ + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + PyThreadState *gilstate = PyGILState_GetThisThreadState(); + + PyThreadStateToken *token1 = PyThreadState_Ensure(guard); + assert(PyThreadState_Get() == gilstate); + + Py_BEGIN_ALLOW_THREADS + assert(PyThreadState_GetUnchecked() == NULL); + PyThreadStateToken *token2 = PyThreadState_Ensure(guard); + assert(PyThreadState_Get() == gilstate); + PyThreadState_Release(token2); + assert(PyThreadState_GetUnchecked() == NULL); + Py_END_ALLOW_THREADS + assert(PyThreadState_Get() == gilstate); + + PyThreadState_Release(token1); + assert(PyThreadState_Get() == gilstate); + + PyInterpreterGuard_Close(guard); + + Py_RETURN_NONE; +} + +/* A capsule destructor that calls Ensure/Release while the tstate is being + * cleared by PyThreadState_Release. */ +static void +tstate_ensure_capsule_destructor(PyObject *capsule) +{ + assert(capsule != NULL); + PyInterpreterGuard *guard = PyCapsule_GetPointer(capsule, "x"); + PyThreadStateToken *token = PyThreadState_Ensure(guard); + assert(token != NULL); + PyThreadState_Release(token); +} + +static PyObject * +test_thread_state_release_with_destructor(PyObject *self, PyObject *unused) +{ + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + assert(guard != NULL); + + // We need to use a fresh thread state in order to control the lifetime of + // it. If we used the current thread state, it wouldn't be cleared until + // the end of the program, which is after the guard has been closed. + PyThreadState *fresh_tstate = PyThreadState_New(PyInterpreterState_Get()); + assert(fresh_tstate != NULL); + + PyThreadState *save_tstate = PyThreadState_Swap(fresh_tstate); + assert(save_tstate != NULL); + + /* Triggers fresh tstate path */ + PyThreadStateToken *token = PyThreadState_Ensure(guard); + assert(token != NULL); + + /* Stash a capsule whose destructor will run during PyThreadState_Clear. */ + PyObject *capsule = PyCapsule_New(guard, "x", tstate_ensure_capsule_destructor); + assert(capsule != NULL); + + /* We need to put it somewhere it gets cleaned up at PyThreadState_Clear. + * tstate->dict is cleared during PyThreadState_Clear. */ + PyObject *dict = PyThreadState_GetDict(); + assert(dict != NULL); + int res = PyDict_SetItemString(dict, "key", capsule); + assert(res == 0); + Py_DECREF(capsule); + + PyThreadState_Release(token); + + // This will trigger the destructor + PyThreadState_Clear(fresh_tstate); + PyThreadState_DeleteCurrent(); + + PyInterpreterGuard_Close(guard); + PyThreadState_Swap(save_tstate); + + Py_RETURN_NONE; +} + static PyObject* test_soft_deprecated_macros(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) @@ -2642,6 +2973,31 @@ test_soft_deprecated_macros(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args) Py_RETURN_NONE; } + +#ifdef HAVE_SYSCTLBYNAME +static PyObject* +uptime_bsd(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + struct timeval tv; + size_t size = sizeof(tv); + int res = sysctlbyname("kern.boottime", &tv, &size, NULL, 0); + if (res != 0) { + return PyErr_SetFromErrno(PyExc_OSError); + } + double boottime = (double)tv.tv_sec + tv.tv_usec * 1e-6; + + PyTime_t now_t; + if (PyTime_Time(&now_t) < 0) { + return NULL; + } + double now = PyTime_AsSecondsDouble(now_t); + + double uptime = now - boottime; + return PyFloat_FromDouble(uptime); +} +#endif + + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -2740,6 +3096,17 @@ static PyMethodDef TestMethods[] = { {"create_managed_weakref_nogc_type", create_managed_weakref_nogc_type, METH_NOARGS}, {"test_soft_deprecated_macros", test_soft_deprecated_macros, METH_NOARGS}, + {"test_interpreter_guards", test_interpreter_guards, METH_NOARGS}, + {"test_thread_state_ensure_nested", test_thread_state_ensure_nested, METH_NOARGS}, + {"test_thread_state_ensure_crossinterp", test_thread_state_ensure_crossinterp, METH_NOARGS}, + {"test_interp_view_after_shutdown", test_interp_view_after_shutdown, METH_NOARGS}, + {"test_thread_state_ensure_view", test_thread_state_ensure_view, METH_NOARGS}, + {"test_thread_state_ensure_detachment", test_thread_state_ensure_detachment, METH_NOARGS}, + {"test_thread_state_ensure_detached_gilstate", test_thread_state_ensure_detached_gilstate, METH_NOARGS}, + {"test_thread_state_release_with_destructor", test_thread_state_release_with_destructor, METH_NOARGS}, +#ifdef HAVE_SYSCTLBYNAME + {"uptime_bsd", uptime_bsd, METH_NOARGS}, +#endif {NULL, NULL} /* sentinel */ }; @@ -3573,6 +3940,9 @@ _testcapi_exec(PyObject *m) if (_PyTestCapi_Init_Module(m) < 0) { return -1; } + if (_PyTestCapi_Init_Weakref(m) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a07675bb66d8cc8..e3de9006d5a427f 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -47,6 +47,9 @@ #if defined(HAVE_DLADDR) && !defined(__wasi__) # include <dlfcn.h> #endif +#if defined(HAVE_EXECINFO_H) +# include <execinfo.h> +#endif #ifdef MS_WINDOWS # include <windows.h> # include <intrin.h> @@ -58,6 +61,47 @@ static const uintptr_t min_frame_pointer_addr = 0x1000; +#define MAX_UNWIND_FRAMES 200 + +#ifdef __s390x__ +// Linux's s390 "Stack Frame Layout" table documents that z/Architecture +// backchain frames start with the backchain at offset 0 and store "saved r14 +// of caller function" at offset 112. The same document's register table +// identifies r14 as the return-address register, so this backchain unwinder +// reads the return address from fp + 112. +// https://www.kernel.org/doc/html/v5.3/s390/debugging390.html#stack-frame-layout +// +// This is only for Linux s390x backchain frames. The s390x ELF ABI does not +// generally mandate where RA and FP are saved, or whether they are saved at all. +// https://sourceware.org/binutils/docs/sframe-spec.html#s390x +# define S390X_FRAME_RETURN_ADDRESS_OFFSET 112 +#endif + +// The generic manual unwinder treats the frame pointer as a two-word record: +// fp[0] is the previous frame pointer and fp[1] is the return address. That is +// not true for every architecture, even with frame pointers enabled, so these +// offsets describe the actual slots used by each supported frame layout. +#if defined(__arm__) && !defined(__thumb__) && !defined(__clang__) +// GCC ARM mode keeps the caller's fp one word below fp and the saved LR at +// fp[0], so the return address is not in the generic fp[1] slot. +# define FRAME_POINTER_NEXT_OFFSET (-1) +# define FRAME_POINTER_RETURN_OFFSET 0 +#elif defined(__s390x__) +// s390x backchain frames keep the previous frame pointer at fp[0], but save the +// return-address register in the ABI register save area rather than fp[1]. +# define FRAME_POINTER_NEXT_OFFSET 0 +# define FRAME_POINTER_RETURN_OFFSET \ + (S390X_FRAME_RETURN_ADDRESS_OFFSET / (Py_ssize_t)sizeof(uintptr_t)) +#elif defined(__powerpc64__) || defined(__ppc64__) +// ppc64le puts the return address at fp[2]; it saves the Condition Register +// in fp[1]. See: +// https://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi-1.9.html#STACK +# define FRAME_POINTER_NEXT_OFFSET 0 +# define FRAME_POINTER_RETURN_OFFSET 2 +#else +# define FRAME_POINTER_NEXT_OFFSET 0 +# define FRAME_POINTER_RETURN_OFFSET 1 +#endif static PyObject * @@ -196,10 +240,17 @@ classify_address(uintptr_t addr, int jit_enabled, PyInterpreterState *interp) if (strncmp(base, "python", 6) == 0) { return "python"; } +#ifdef __CYGWIN__ + // Match Cygwin "cygpython3.16.dll" + if (strncmp(base, "cygpython", 9) == 0) { + return "python"; + } +#else // Match "libpython3.15.so.1.0" if (strncmp(base, "libpython", 9) == 0) { return "python"; } +#endif return "other"; } #ifdef _Py_JIT @@ -325,30 +376,130 @@ get_jit_backend(PyObject *self, PyObject *Py_UNUSED(args)) #endif } +static int +stack_address_is_valid(uintptr_t addr, uintptr_t stack_min, uintptr_t stack_max) +{ + if (addr < min_frame_pointer_addr) { + return 0; + } + if (stack_min != 0 && (addr < stack_min || addr >= stack_max)) { + return 0; + } + return 1; +} + +static int +frame_pointer_slot_is_valid(uintptr_t *frame_pointer, Py_ssize_t offset, + uintptr_t stack_min, uintptr_t stack_max) +{ + uintptr_t fp_addr = (uintptr_t)frame_pointer; + uintptr_t slot_addr; + uintptr_t delta = (uintptr_t)Py_ABS(offset) * sizeof(uintptr_t); + if (offset < 0) { + if (fp_addr < delta) { + return 0; + } + slot_addr = fp_addr - delta; + } + else { + if (fp_addr > UINTPTR_MAX - delta) { + return 0; + } + slot_addr = fp_addr + delta; + } + if (!stack_address_is_valid(slot_addr, stack_min, stack_max)) { + return 0; + } + if (stack_max != 0) { + if (slot_addr > UINTPTR_MAX - sizeof(uintptr_t)) { + return 0; + } + if (slot_addr + sizeof(uintptr_t) > stack_max) { + return 0; + } + } + return 1; +} + +static int +next_frame_pointer_is_valid(uintptr_t *frame_pointer, uintptr_t *next_fp, + uintptr_t stack_min, uintptr_t stack_max) +{ + uintptr_t fp_addr = (uintptr_t)frame_pointer; + uintptr_t next_addr = (uintptr_t)next_fp; + if (!stack_address_is_valid(next_addr, stack_min, stack_max)) { + return 0; + } + if ((next_addr % sizeof(uintptr_t)) != 0) { + return 0; + } +#if _Py_STACK_GROWS_DOWN + return next_addr > fp_addr; +#else + return next_addr < fp_addr; +#endif +} + static PyObject * manual_unwind_from_fp(uintptr_t *frame_pointer) { - Py_ssize_t max_depth = 200; - int stack_grows_down = _Py_STACK_GROWS_DOWN; + uintptr_t stack_min = 0; + uintptr_t stack_max = 0; + +#ifdef __s390x__ + Py_BUILD_ASSERT(S390X_FRAME_RETURN_ADDRESS_OFFSET % sizeof(uintptr_t) == 0); +#endif if (frame_pointer == NULL) { return PyList_New(0); } + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate != NULL) { + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN + stack_min = tstate_impl->c_stack_hard_limit; + stack_max = tstate_impl->c_stack_top; +#else + stack_min = tstate_impl->c_stack_top; + stack_max = tstate_impl->c_stack_hard_limit; +#endif + } + PyObject *result = PyList_New(0); if (result == NULL) { return NULL; } - for (Py_ssize_t depth = 0; - depth < max_depth && frame_pointer != NULL; - depth++) - { + Py_ssize_t depth = 0; + while (frame_pointer != NULL) { uintptr_t fp_addr = (uintptr_t)frame_pointer; if ((fp_addr % sizeof(uintptr_t)) != 0) { break; } - uintptr_t return_addr = frame_pointer[1]; + if (depth >= MAX_UNWIND_FRAMES) { + Py_DECREF(result); + PyErr_Format( + PyExc_RuntimeError, + "manual frame pointer unwind returned more than %d frames", + MAX_UNWIND_FRAMES); + return NULL; + } + if (!stack_address_is_valid(fp_addr, stack_min, stack_max)) { + break; + } + if (!frame_pointer_slot_is_valid(frame_pointer, + FRAME_POINTER_NEXT_OFFSET, + stack_min, stack_max)) { + break; + } + if (!frame_pointer_slot_is_valid(frame_pointer, + FRAME_POINTER_RETURN_OFFSET, + stack_min, stack_max)) { + break; + } + uintptr_t *next_fp = (uintptr_t *)frame_pointer[FRAME_POINTER_NEXT_OFFSET]; + uintptr_t return_addr = frame_pointer[FRAME_POINTER_RETURN_OFFSET]; PyObject *addr_obj = PyLong_FromUnsignedLongLong(return_addr); if (addr_obj == NULL) { @@ -361,28 +512,60 @@ manual_unwind_from_fp(uintptr_t *frame_pointer) return NULL; } Py_DECREF(addr_obj); + depth++; - uintptr_t *next_fp = (uintptr_t *)frame_pointer[0]; - // Stop if the frame pointer is extremely low. - if ((uintptr_t)next_fp < min_frame_pointer_addr) { + if (!next_frame_pointer_is_valid(frame_pointer, next_fp, + stack_min, stack_max)) { break; } - uintptr_t next_addr = (uintptr_t)next_fp; - if (stack_grows_down) { - if (next_addr <= fp_addr) { - break; - } - } - else { - if (next_addr >= fp_addr) { - break; - } - } frame_pointer = next_fp; } return result; } + +#if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) +static PyObject * +gnu_backtrace_unwind(PyObject *self, PyObject *Py_UNUSED(args)) +{ + void *addresses[MAX_UNWIND_FRAMES + 1]; + int frame_count = backtrace(addresses, (int)Py_ARRAY_LENGTH(addresses)); + if (frame_count < 0) { + PyErr_SetString(PyExc_RuntimeError, "backtrace() failed"); + return NULL; + } + if (frame_count > MAX_UNWIND_FRAMES) { + PyErr_Format( + PyExc_RuntimeError, + "backtrace() returned more than %d frames", + MAX_UNWIND_FRAMES); + return NULL; + } + + PyObject *result = PyList_New(frame_count); + if (result == NULL) { + return NULL; + } + for (int i = 0; i < frame_count; i++) { + PyObject *addr_obj = PyLong_FromUnsignedLongLong((uintptr_t)addresses[i]); + if (addr_obj == NULL) { + Py_DECREF(result); + return NULL; + } + PyList_SET_ITEM(result, i, addr_obj); + } + return result; +} +#else +static PyObject * +gnu_backtrace_unwind(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyErr_SetString(PyExc_RuntimeError, + "gnu_backtrace_unwind is not supported on this platform"); + return NULL; +} +#endif + #if defined(__GNUC__) || defined(__clang__) static PyObject * manual_frame_pointer_unwind(PyObject *self, PyObject *args) @@ -929,7 +1112,7 @@ static PyObject * set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args)) { module_state *state = get_module_state(self); - _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), _PyEval_EvalFrameDefault); + _PyInterpreterState_SetEvalFrameFunc(_PyInterpreterState_GET(), NULL); Py_CLEAR(state->record_list); Py_RETURN_NONE; } @@ -1736,7 +1919,7 @@ pending_identify(PyObject *self, PyObject *args) PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { - return NULL; + return PyErr_NoMemory(); } PyThread_acquire_lock(mutex, WAIT_LOCK); /* It gets released in _pending_identify_callback(). */ @@ -2731,8 +2914,7 @@ has_deferred_refcount(PyObject *self, PyObject *op) static PyObject * get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) { - // Generational GC doesn't track heap_size, return -1. - return PyLong_FromInt64(-1); + return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); } static PyObject * @@ -2890,6 +3072,69 @@ test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +#define NUM_GUARDS 100 + +static PyObject * +test_interp_guard_countdown(PyObject *self, PyObject *unused) +{ + PyThreadState *save_tstate = PyThreadState_Swap(NULL); + + // This test assumes that the interpreter has no guards active. + // While this is currently true for the main interpreter as of writing, + // this won't necessarily be true in the future. For the sake of + // maintainance, we create a new interpreter to be sure that there aren't + // any other guards. + PyThreadState *interp_tstate = Py_NewInterpreter(); + assert(interp_tstate != NULL); + PyInterpreterState *interp = PyInterpreterState_Get(); + assert(_PyInterpreterState_GuardCountdown(interp) == 0); + + PyInterpreterGuard *guards[NUM_GUARDS]; + for (int i = 0; i < NUM_GUARDS; ++i) { + guards[i] = PyInterpreterGuard_FromCurrent(); + assert(guards[i] != 0); + assert(_PyInterpreterState_GuardCountdown(interp) == i + 1); + } + + for (int i = 0; i < NUM_GUARDS; ++i) { + PyInterpreterGuard_Close(guards[i]); + assert(_PyInterpreterState_GuardCountdown(interp) == (NUM_GUARDS - i - 1)); + } + + Py_EndInterpreter(interp_tstate); + PyThreadState_Swap(save_tstate); + Py_RETURN_NONE; +} + +static PyObject * +test_interp_view_countdown(PyObject *self, PyObject *unused) +{ + PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + if (view == NULL) { + return NULL; + } + assert(_PyInterpreterState_GuardCountdown(interp) == 0); + + PyInterpreterGuard *guards[NUM_GUARDS]; + + for (int i = 0; i < NUM_GUARDS; ++i) { + guards[i] = PyInterpreterGuard_FromView(view); + assert(guards[i] != 0); + assert(_PyInterpreterGuard_GetInterpreter(guards[i]) == interp); + assert(_PyInterpreterState_GuardCountdown(interp) == i + 1); + } + + for (int i = 0; i < NUM_GUARDS; ++i) { + PyInterpreterGuard_Close(guards[i]); + assert(_PyInterpreterState_GuardCountdown(interp) == (NUM_GUARDS - i - 1)); + } + + PyInterpreterView_Close(view); + Py_RETURN_NONE; +} + +#undef NUM_LOCKS static PyObject * _pyerr_setkeyerror(PyObject *self, PyObject *arg) @@ -2904,6 +3149,52 @@ _pyerr_setkeyerror(PyObject *self, PyObject *arg) return NULL; } +static PyObject * +test_thread_state_ensure_from_view_interp_switch(PyObject *self, PyObject *unused) +{ + /* The main tstate is already attached and was NOT created by + * PyThreadState_Ensure, so delete_on_release == 0. */ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp != NULL); + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + assert(view != NULL); + + /* First Ensure/Release pair on this pre-existing tstate. */ + assert(_PyThreadState_GET() != NULL); + PyThreadStateToken *t1 = PyThreadState_EnsureFromView(view); + assert(t1 != NULL); + assert(_PyInterpreterState_GuardCountdown(interp) == 1); + PyThreadState_Release(t1); + assert(_PyInterpreterState_GuardCountdown(interp) == 0); + assert(_PyThreadState_GET() != NULL); + + /* tstate->ensure.owned_guard now points at the freed guard. */ + + /* Re-attach: Bug B detaches us as a side effect (separate repro). */ + PyThreadState *save = PyThreadState_Swap(NULL); + + PyThreadStateToken *t2 = PyThreadState_EnsureFromView(view); + assert(_PyInterpreterState_GuardCountdown(interp) == 1); + assert(t2 != NULL); + PyThreadState_Release(t2); + assert(_PyInterpreterState_GuardCountdown(interp) == 0); + assert(_PyThreadState_GET() == NULL); + + PyThreadState_Swap(save); + + /* In a release build (no assertion) the second Ensure silently + * skipped storing its guard and Release decremented the global + * counter from 0, wrapping it to GUARDS_NOT_ALLOWED. All future + * guard acquisitions then fail: */ + PyInterpreterGuard *g = PyInterpreterGuard_FromCurrent(); + assert(g != NULL); + assert(_PyInterpreterState_GuardCountdown(interp) == 1); + PyInterpreterGuard_Close(g); + assert(_PyInterpreterState_GuardCountdown(interp) == 0); + + PyInterpreterView_Close(view); + Py_RETURN_NONE; +} static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, @@ -2915,6 +3206,7 @@ static PyMethodDef module_functions[] = { {"classify_stack_addresses", classify_stack_addresses, METH_VARARGS}, {"get_jit_code_ranges", get_jit_code_ranges, METH_NOARGS}, {"get_jit_backend", get_jit_backend, METH_NOARGS}, + {"gnu_backtrace_unwind", gnu_backtrace_unwind, METH_NOARGS}, {"manual_frame_pointer_unwind", manual_frame_pointer_unwind, METH_NOARGS}, {"test_bswap", test_bswap, METH_NOARGS}, {"test_popcount", test_popcount, METH_NOARGS}, @@ -3028,6 +3320,9 @@ static PyMethodDef module_functions[] = { {"test_threadstate_set_stack_protection", test_threadstate_set_stack_protection, METH_NOARGS}, {"_pyerr_setkeyerror", _pyerr_setkeyerror, METH_O}, + {"test_interp_guard_countdown", test_interp_guard_countdown, METH_NOARGS}, + {"test_interp_view_countdown", test_interp_view_countdown, METH_NOARGS}, + {"test_thread_state_ensure_from_view_interp_switch", test_thread_state_ensure_from_view_interp_switch, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -3117,6 +3412,12 @@ module_exec(PyObject *module) return 1; } +#ifdef _Py_WITH_FRAME_POINTERS + if (PyModule_AddIntMacro(module, _Py_WITH_FRAME_POINTERS) < 0) { + return 1; + } +#endif + return 0; } diff --git a/Modules/_testinternalcapi/interpreter.c b/Modules/_testinternalcapi/interpreter.c index 99dcd18393fb870..237635c5f2bcdba 100644 --- a/Modules/_testinternalcapi/interpreter.c +++ b/Modules/_testinternalcapi/interpreter.c @@ -18,8 +18,13 @@ int Test_EvalFrame_Resumes, Test_EvalFrame_Loads; static int stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) { - (void)(tstate); (void)(frame); + // Don't actually JIT-compile in this test eval-frame, but we still must + // finalize the tracer so the thread-global is_tracing flag is reset. + // Otherwise a trace started inside this duplicated interpreter loop + // (reachable under low JIT thresholds, e.g. PYTHON_JIT_STRESS=1) would + // leave is_tracing stuck true and permanently disable the JIT. + _PyJit_FinalizeTracing(tstate, 0); return 0; } #endif @@ -87,6 +92,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) entry.frame.return_offset = 0; #ifdef Py_DEBUG entry.frame.lltrace = 0; + entry.frame.stackpointer_valid = 1; #endif /* Push frame */ entry.frame.previous = tstate->current_frame; @@ -117,6 +123,7 @@ Test_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) next_instr = frame->instr_ptr; monitor_throw(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index dac46a3e22afa9b..76493327ca0a00b 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -45,8 +45,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); @@ -62,8 +63,9 @@ PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -76,9 +78,10 @@ value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -86,8 +89,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -344,10 +348,11 @@ assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); assert(d != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int match = (d->guard != NULL) ? d->guard(left_o, right_o) : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!match) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -363,9 +368,10 @@ assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -381,9 +387,10 @@ value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -391,8 +398,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -463,9 +471,13 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyUnicode_Append(&temp, right_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); *target_local = PyStackRef_NULL; if (temp == NULL) { JUMP_TO_LABEL(error); @@ -670,8 +682,9 @@ assert(Py_TYPE(dict)->tp_as_mapping->mp_subscript == _PyDict_Subscript); STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_Subscript(dict, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -684,9 +697,10 @@ value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ds; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -694,8 +708,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -773,12 +788,18 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -837,8 +858,9 @@ } #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -872,8 +894,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -927,8 +950,9 @@ assert(PySlice_Check(sub)); assert(PyList_CheckExact(list)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); STAT_INC(BINARY_OP, hit); if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -942,9 +966,10 @@ value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ls; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -952,8 +977,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -1133,8 +1159,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -1405,18 +1432,21 @@ PyObject *res_o; if (PyList_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyList_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyTuple_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyUnicode_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { PyObject *slice = PySlice_New(start_o, stop_o, NULL); @@ -1425,25 +1455,36 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_GetItem(container_o, slice); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = stop; stop = PyStackRef_NULL; stack_pointer[-1] = stop; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = start; start = PyStackRef_NULL; stack_pointer[-2] = start; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -1483,14 +1524,16 @@ format_o = &_Py_STR(empty); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer += -(oparg & 1); @@ -1498,13 +1541,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (interpolation_o == NULL) { JUMP_TO_LABEL(error); } @@ -1527,8 +1572,9 @@ _PyStackRef list; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (list_o == NULL) { JUMP_TO_LABEL(error); } @@ -1551,8 +1597,9 @@ _PyStackRef map; values = &stack_pointer[-oparg*2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (map_o == NULL) { stack_pointer += -oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1577,17 +1624,19 @@ _PyStackRef set; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = values[_i]; values[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -1597,22 +1646,25 @@ _PyStackRef value = values[i]; values[i] = PyStackRef_NULL; if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } if (err) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectStealMortal(set_o); @@ -1638,13 +1690,14 @@ PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; args[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { @@ -1669,8 +1722,9 @@ _PyStackRef str; pieces = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (str_o == NULL) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1699,18 +1753,21 @@ PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (template_o == NULL) { JUMP_TO_LABEL(error); } @@ -1782,8 +1839,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -1803,8 +1861,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL @@ -1826,11 +1885,12 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -1842,6 +1902,7 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -1851,7 +1912,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1865,8 +1926,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -1951,8 +2013,9 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (self_o == NULL) { JUMP_TO_LABEL(error); } @@ -1961,32 +2024,36 @@ callable = PyStackRef_FromPyObjectNew(init_func); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CREATE_INIT_FRAME { args = &stack_pointer[-oparg]; self = self_or_null; init = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -1999,12 +2066,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2062,8 +2135,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_FUNCTION_VERSION @@ -2142,12 +2216,18 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2219,8 +2299,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_RECURSION_REMAINING @@ -2243,12 +2324,13 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -2270,12 +2352,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2326,27 +2414,30 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_CallBuiltinClass_StackRef( callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2354,14 +2445,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2413,28 +2506,31 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFast_StackRef( callable, arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2442,14 +2538,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2501,24 +2599,27 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2526,14 +2627,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2605,8 +2708,9 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -2624,8 +2728,9 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2633,14 +2738,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2684,23 +2791,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _CALL_FUNCTION_EX_NON_PY_GENERAL @@ -2717,23 +2827,27 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -2745,8 +2859,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2788,23 +2903,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _CHECK_IS_PY_CALLABLE_EX @@ -2839,10 +2957,11 @@ stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -2865,12 +2984,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2904,8 +3029,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallFunctionEx(func, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_FUNCTION_EX); @@ -2918,23 +3044,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX @@ -2957,34 +3086,39 @@ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } @@ -3003,10 +3137,11 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -3022,24 +3157,28 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -3051,8 +3190,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -3076,8 +3216,9 @@ value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3088,9 +3229,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3117,8 +3259,9 @@ PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3131,9 +3274,10 @@ value = vs2; stack_pointer[-2] = res; stack_pointer[-1] = vs1; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3141,8 +3285,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3193,8 +3338,9 @@ PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (retval < 0) { JUMP_TO_LABEL(error); } @@ -3202,18 +3348,21 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); } @@ -3250,8 +3399,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -3271,8 +3421,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_KW @@ -3297,16 +3448,18 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -3317,6 +3470,7 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -3326,7 +3480,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3407,8 +3561,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _PY_FRAME_KW @@ -3427,17 +3582,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -3459,12 +3616,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -3519,12 +3682,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3538,8 +3702,9 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -3615,16 +3780,18 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -3646,12 +3813,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -3703,8 +3876,9 @@ STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } @@ -3725,8 +3899,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3734,8 +3909,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3775,8 +3951,7 @@ // _GUARD_NOS_NOT_NULL { nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { + if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -3820,8 +3995,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3829,8 +4005,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3900,7 +4077,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, @@ -3908,22 +4089,24 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3931,14 +4114,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4012,7 +4197,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, @@ -4020,22 +4209,24 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4043,14 +4234,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4133,8 +4326,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4152,8 +4346,9 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4161,14 +4356,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4252,8 +4449,9 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4273,8 +4471,9 @@ stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4282,8 +4481,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4291,14 +4491,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4353,12 +4555,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -4372,8 +4575,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4483,12 +4687,18 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -4557,11 +4767,12 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -4583,12 +4794,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -4639,8 +4856,9 @@ assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4654,14 +4872,16 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4714,8 +4934,9 @@ assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4729,14 +4950,16 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4798,8 +5021,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -4821,37 +5045,49 @@ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { @@ -4863,8 +5099,9 @@ } if (!Py_IsNone(match_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -4892,19 +5129,22 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyExceptionInstance_Check(left_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; @@ -4939,37 +5179,55 @@ #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub_iter; sub_iter = value; stack_pointer[-4] = sub_iter; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-1] = exc_value_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = last_sent_val; last_sent_val = PyStackRef_NULL; stack_pointer[-2] = last_sent_val; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = null_in; null_in = PyStackRef_NULL; stack_pointer[-3] = null_in; PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); null_out = null_in; none = PyStackRef_None; } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; @@ -5004,8 +5262,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); @@ -5018,16 +5277,23 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = right; right = PyStackRef_NULL; stack_pointer[-1] = right; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = left; left = PyStackRef_NULL; stack_pointer[-2] = left; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -5035,9 +5301,13 @@ } if (oparg & 16) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res_bool = PyObject_IsTrue(res_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_bool < 0) { JUMP_TO_LABEL(error); } @@ -5294,8 +5564,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); @@ -5308,8 +5579,9 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5322,9 +5594,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5332,8 +5605,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5376,8 +5650,9 @@ assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5390,9 +5665,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5400,8 +5676,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5444,8 +5721,9 @@ assert(PyAnySet_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5458,9 +5736,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5468,8 +5747,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5489,13 +5769,15 @@ assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -5557,13 +5839,15 @@ owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -5582,13 +5866,15 @@ PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5603,18 +5889,20 @@ _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5628,16 +5916,18 @@ INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } DISPATCH(); @@ -5656,20 +5946,23 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } DISPATCH(); @@ -5688,17 +5981,24 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -5730,13 +6030,18 @@ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); PyObject *dupkey = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } u = update; @@ -5747,8 +6052,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5772,29 +6078,37 @@ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (has_keys == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'%T' object is not a mapping", update_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_ChainExceptions1(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } JUMP_TO_LABEL(error); @@ -5807,8 +6121,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5832,27 +6147,39 @@ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } DISPATCH(); @@ -5870,8 +6197,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5896,8 +6224,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5963,10 +6292,11 @@ should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; @@ -6004,13 +6334,15 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6040,16 +6372,23 @@ fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fmt_spec; fmt_spec = PyStackRef_NULL; stack_pointer[-1] = fmt_spec; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = value; value = PyStackRef_NULL; stack_pointer[-2] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6086,8 +6425,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -6097,8 +6437,9 @@ // _FOR_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -6168,12 +6509,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -6238,8 +6585,9 @@ _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); @@ -6397,15 +6745,14 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; _PyStackRef null_or_index; + _PyStackRef iter; _PyStackRef next; /* Skip 1 cache entry */ - // _GUARD_NOS_ITER_VIRTUAL + // _GUARD_TOS_NOT_NULL { - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o)->_tp_iteritem == NULL) { + null_or_index = stack_pointer[-1]; + if (PyStackRef_IsNull(null_or_index)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); @@ -6413,12 +6760,13 @@ } // _FOR_ITER_VIRTUAL { - null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { @@ -6458,38 +6806,46 @@ } if (getter == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -6511,8 +6867,9 @@ _PyStackRef awaitable; aiter = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (awaitable_o == NULL) { JUMP_TO_LABEL(error); } @@ -6535,13 +6892,15 @@ _PyStackRef iter; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } @@ -6575,8 +6934,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_GetIter(iterable, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(GET_ITER); @@ -6586,8 +6946,9 @@ // _GET_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsError(result)) { JUMP_TO_LABEL(pop_1_error); } @@ -6682,8 +7043,9 @@ _PyStackRef len; obj = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } @@ -6713,15 +7075,17 @@ PyObject *res_o; if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportFrom( tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportFrom( tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -6750,31 +7114,37 @@ PyObject *res_o; if (!(oparg & 0x02)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level), oparg & 0x01); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = level; level = PyStackRef_NULL; stack_pointer[-2] = level; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6819,8 +7189,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL @@ -6843,11 +7214,12 @@ stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg0 ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -6870,12 +7242,13 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -6884,7 +7257,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -6894,7 +7268,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6908,8 +7282,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -6943,23 +7318,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX @@ -6982,34 +7360,39 @@ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } @@ -7028,10 +7411,11 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -7047,24 +7431,28 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -7076,8 +7464,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7117,8 +7506,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL_KW @@ -7139,10 +7529,11 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7166,17 +7557,19 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -7184,7 +7577,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -7194,7 +7588,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -7234,27 +7628,39 @@ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } } @@ -7276,8 +7682,9 @@ receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7285,8 +7692,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7310,8 +7718,9 @@ PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7322,8 +7731,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7344,8 +7754,9 @@ null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -7375,9 +7786,10 @@ INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (next_opcode < 0) { JUMP_TO_LABEL(error); } @@ -7404,8 +7816,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7451,9 +7864,13 @@ next_instr = this_instr; } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); original_opcode = _Py_call_instrumentation_line( tstate, frame, this_instr, prev_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); if (original_opcode < 0) { next_instr = this_instr+1; JUMP_TO_LABEL(error); @@ -7468,6 +7885,7 @@ PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } @@ -7499,25 +7917,33 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -7527,45 +7953,56 @@ { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { @@ -7573,9 +8010,13 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -7630,8 +8071,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7681,8 +8123,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } stack_pointer += -1; @@ -7709,8 +8152,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { @@ -7760,9 +8204,10 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } @@ -7787,8 +8232,9 @@ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7802,8 +8248,9 @@ Test_EvalFrame_Resumes++; if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7812,9 +8259,13 @@ // _MONITOR_RESUME { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation( tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7843,10 +8294,11 @@ { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7864,12 +8316,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -7890,6 +8345,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; _PyStackRef val; _PyStackRef value; _PyStackRef retval; @@ -7897,14 +8353,21 @@ { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); next_instr = frame->instr_ptr; DISPATCH(); } @@ -7926,24 +8389,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -7976,12 +8443,14 @@ _PyStackRef executor = frame->localsplus[0]; assert(tstate->current_executor == NULL); if (!PyStackRef_IsNull(executor)) { + assert(PyStackRef_TYPE(executor) == &_PyUOpExecutor_Type); tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } #endif @@ -8018,8 +8487,9 @@ stack_pointer[-2] = b; stack_pointer[-1] = l; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -8027,8 +8497,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -8059,8 +8530,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8088,8 +8560,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8156,8 +8629,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8223,21 +8697,24 @@ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } @@ -8250,8 +8727,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -8280,8 +8758,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); @@ -8295,22 +8774,25 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(pop_1_error); } } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(error); } @@ -8363,11 +8845,12 @@ assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { @@ -8431,11 +8914,12 @@ assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { @@ -8527,12 +9011,18 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -8609,8 +9099,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -8765,18 +9256,7 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); @@ -8833,7 +9313,7 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_value); if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); @@ -8859,8 +9339,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -8912,8 +9393,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; @@ -8959,18 +9441,7 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } + /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); @@ -8980,8 +9451,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; @@ -9063,12 +9535,18 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -9132,8 +9610,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9249,8 +9728,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9276,16 +9756,18 @@ _PyStackRef bc; int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); @@ -9305,7 +9787,7 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -9341,15 +9823,17 @@ _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer[0] = value; @@ -9442,11 +9926,12 @@ _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); @@ -9495,8 +9980,9 @@ name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -9504,17 +9990,19 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; @@ -9536,13 +10024,15 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(mod_or_class_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -9550,58 +10040,71 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } @@ -9637,8 +10140,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); @@ -9653,8 +10157,9 @@ res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(*res)) { JUMP_TO_LABEL(error); } @@ -9720,7 +10225,7 @@ } assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); @@ -9787,7 +10292,7 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); @@ -9830,9 +10335,10 @@ PyObject *l = LOCALS(); if (l == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); @@ -9853,34 +10359,43 @@ _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (l_v == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -9931,21 +10446,24 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err <= 0) { if (err == 0) { PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) ? _Py_SpecialMethods[oparg].error_suggestion : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(!_PyErr_Occurred(tstate)); assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } @@ -9981,8 +10499,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -9998,25 +10517,33 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -10026,45 +10553,56 @@ { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { @@ -10072,9 +10610,13 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -10129,20 +10671,30 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { @@ -10203,9 +10755,10 @@ { int *method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (attr_o == NULL) { JUMP_TO_LABEL(error); @@ -10216,23 +10769,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); self_or_null = PyStackRef_NULL; stack_pointer += 1; } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-1] = class_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -10260,8 +10818,9 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10282,9 +10841,10 @@ codeobj_st = stack_pointer[-1]; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (func_obj == NULL) { JUMP_TO_LABEL(error); } @@ -10297,9 +10857,10 @@ { value = co; stack_pointer[-1] = func; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -10321,12 +10882,13 @@ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(pop_2_error); } @@ -10358,11 +10920,12 @@ subject = stack_pointer[-3]; assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -10383,9 +10946,10 @@ stack_pointer[-3] = attrs; stack_pointer[-2] = s; stack_pointer[-1] = tp; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -10393,8 +10957,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -10402,8 +10967,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -10422,9 +10988,10 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (values_or_none_o == NULL) { JUMP_TO_LABEL(error); } @@ -10507,10 +11074,11 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); @@ -10532,8 +11100,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10582,11 +11151,12 @@ else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_TRUE @@ -10625,11 +11195,12 @@ else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_FALSE @@ -10680,8 +11251,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10748,12 +11320,17 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } JUMP_TO_LABEL(error); @@ -10782,8 +11359,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } @@ -10817,9 +11401,10 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } @@ -10844,8 +11429,9 @@ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -10859,16 +11445,18 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Resume(this_instr, tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { Test_EvalFrame_Resumes++; if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -10998,13 +11586,15 @@ assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); } assert(STACK_LEVEL() <= 2); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); @@ -11013,10 +11603,12 @@ gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -11049,12 +11641,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -11089,8 +11684,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); @@ -11102,7 +11698,6 @@ v = stack_pointer[-1]; null_or_index = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && @@ -11121,10 +11716,11 @@ gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } - if (!PyStackRef_IsNull(null_or_index)) { + if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -11136,51 +11732,93 @@ retval = item; } else { - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PySendResultPair res = _PyIter_Send(receiver_o, v_o); + _PyFrame_StackPointerInvalidate(frame); + if (res.kind == PYGEN_ERROR) { + JUMP_TO_LABEL(error); } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + retval = PyStackRef_FromPyObjectSteal(res.object); + if (res.kind == PYGEN_RETURN) { + JUMPBY(oparg); } - retval = PyStackRef_FromPyObjectSteal(retval_o); + stack_pointer += 1; } - stack_pointer[-2] = null_or_index; - stack_pointer[-1] = retval; + } + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = retval; + DISPATCH(); + } + + TARGET(SEND_ASYNC_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_ASYNC_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_ASYNC_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_in; + _PyStackRef v; + _PyStackRef asend; + _PyStackRef null_out; + _PyStackRef retval; + /* Skip 1 cache entry */ + // _GUARD_3OS_ASYNC_GEN_ASEND + { + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyAsyncGenASend_CheckExact(iter_o)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_ASYNC_GEN + { + v = stack_pointer[-1]; + null_in = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyAsyncGenASend_CheckExact(iter_o)); + PyObject *val = PyStackRef_AsPyObjectBorrow(v); + PyObject *retval_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); + _PyFrame_StackPointerInvalidate(frame); + if (what == PYGEN_ERROR) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + asend = iter; + null_out = null_in; + retval = PyStackRef_FromPyObjectSteal(retval_o); + if (what == PYGEN_RETURN) { + JUMPBY(oparg); + } } + stack_pointer[-2] = asend; + stack_pointer[-1] = null_out; + stack_pointer[0] = retval; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11241,17 +11879,88 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } + TARGET(SEND_VIRTUAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_VIRTUAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_VIRTUAL); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef val; + _PyStackRef nos; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef none; + _PyStackRef next; + /* Skip 1 cache entry */ + // _GUARD_TOS_IS_NONE + { + val = stack_pointer[-1]; + if (!PyStackRef_IsNone(val)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + if (PyStackRef_IsNull(nos)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_VIRTUAL + { + none = val; + null_or_index = nos; + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); + _PyFrame_StackPointerInvalidate(frame); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + JUMP_TO_LABEL(error); + } + next = none; + JUMPBY(oparg); + DISPATCH(); + } + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + } + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = next; + DISPATCH(); + } + TARGET(SETUP_ANNOTATIONS) { #if _Py_TAIL_CALL_INTERP int opcode = SETUP_ANNOTATIONS; @@ -11262,38 +11971,46 @@ INSTRUCTION_STATS(SETUP_ANNOTATIONS); if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (ann_dict == NULL) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11311,9 +12028,10 @@ v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(pop_1_error); } @@ -11367,9 +12085,10 @@ iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -11381,8 +12100,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11410,8 +12130,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_ATTR); @@ -11424,17 +12145,24 @@ v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = PyStackRef_NULL; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-2] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -11520,8 +12248,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11529,8 +12258,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11582,8 +12312,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11591,8 +12322,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11671,8 +12403,9 @@ } } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); @@ -11681,8 +12414,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11690,8 +12424,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11708,8 +12443,9 @@ v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); @@ -11738,8 +12474,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11762,8 +12499,9 @@ value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[-1] = value2; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -11786,15 +12524,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); tmp = GETLOCAL(oparg2); GETLOCAL(oparg2) = value2; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -11810,13 +12550,15 @@ v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -11838,31 +12580,36 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -11894,9 +12641,11 @@ container = stack_pointer[-3]; v = stack_pointer[-4]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_AsPyObjectSteal(stop), + Py_None); + _PyFrame_StackPointerInvalidate(frame); int err; if (slice == NULL) { err = 1; @@ -11905,21 +12654,29 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 2; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-4] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -11953,8 +12710,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); @@ -11965,20 +12723,30 @@ { v = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-3] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -12029,16 +12797,18 @@ assert(Py_TYPE(dict)->tp_as_mapping->mp_ass_subscript == _PyDict_StoreSubscript); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } st = dict_st; @@ -12049,8 +12819,9 @@ stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12133,8 +12904,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_INT { @@ -12148,8 +12920,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12196,8 +12969,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); @@ -12208,13 +12982,15 @@ // _TO_BOOL { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -12265,8 +13041,9 @@ value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12379,8 +13156,9 @@ value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12484,30 +13262,34 @@ _PyJitTracerState *tracer = _tstate->jit_tracer_state; assert(tracer != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (full) { LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } DISPATCH(); } for (int i = 0; i < tracer->prev_state.recorded_count; i++) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(tracer->prev_state.recorded_values[i]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.recorded_count = 0; tracer->prev_state.instr = next_instr; PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; @@ -12546,8 +13328,9 @@ { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12558,9 +13341,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12580,8 +13364,9 @@ { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12592,9 +13377,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12633,9 +13419,13 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } @@ -12666,8 +13456,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); @@ -12683,9 +13474,13 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } @@ -12748,8 +13543,9 @@ stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12798,8 +13594,9 @@ stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12849,8 +13646,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12888,13 +13686,15 @@ PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12913,6 +13713,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); + opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; // _MAKE_HEAP_SAFE @@ -12932,24 +13733,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -13002,32 +13807,42 @@ JUMP_TO_LABEL(error); LABEL(error) { + _PyFrame_StackAssertInvalid(frame); #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "error return without exception set"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } #else assert(_PyErr_Occurred(tstate)); #endif _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); STOP_TRACING(); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!_PyFrame_IsIncomplete(frame)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyFrameObject *f = _PyFrame_GetFrameObject(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (f != NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyTraceBack_Here(f); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_MonitorRaise(tstate, frame, next_instr-1); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } @@ -13072,6 +13887,7 @@ JUMP_TO_LABEL(error); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif @@ -13079,6 +13895,13 @@ JUMP_TO_LABEL(error); } LABEL(exit_unwind) + { + assert(_PyErr_Occurred(tstate)); + DTRACE_FUNCTION_RETURN(); + JUMP_TO_LABEL(exit_unwind_notrace); + } + + LABEL(exit_unwind_notrace) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); @@ -13104,15 +13927,20 @@ JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } LABEL(start_frame) { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - JUMP_TO_LABEL(exit_unwind); + JUMP_TO_LABEL(exit_unwind_notrace); } + DTRACE_FUNCTION_ENTRY(); next_instr = frame->instr_ptr; #ifdef Py_DEBUG int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); @@ -13123,6 +13951,7 @@ JUMP_TO_LABEL(error); assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif @@ -13138,12 +13967,14 @@ JUMP_TO_LABEL(error); assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } diff --git a/Modules/_testinternalcapi/test_targets.h b/Modules/_testinternalcapi/test_targets.h index d99b618c8393fcb..1a7eb9169fc837c 100644 --- a/Modules/_testinternalcapi/test_targets.h +++ b/Modules/_testinternalcapi/test_targets.h @@ -202,7 +202,9 @@ static void *opcode_targets_table[256] = { &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_RESUME_CHECK, &&TARGET_RESUME_CHECK_JIT, + &&TARGET_SEND_ASYNC_GEN, &&TARGET_SEND_GEN, + &&TARGET_SEND_VIRTUAL, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, @@ -231,8 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -476,8 +476,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -527,6 +527,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); @@ -724,7 +725,9 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAM static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); @@ -968,7 +971,9 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, [SEND] = _TAIL_CALL_SEND, + [SEND_ASYNC_GEN] = _TAIL_CALL_SEND_ASYNC_GEN, [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SEND_VIRTUAL] = _TAIL_CALL_SEND_VIRTUAL, [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, [SET_ADD] = _TAIL_CALL_SET_ADD, [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, @@ -1014,8 +1019,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [217] = _TAIL_CALL_UNKNOWN_OPCODE, - [218] = _TAIL_CALL_UNKNOWN_OPCODE, [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1226,7 +1229,9 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_ASYNC_GEN] = _TAIL_CALL_TRACE_RECORD, [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SEND_VIRTUAL] = _TAIL_CALL_TRACE_RECORD, [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, [SET_ADD] = _TAIL_CALL_TRACE_RECORD, [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, @@ -1272,8 +1277,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [217] = _TAIL_CALL_UNKNOWN_OPCODE, - [218] = _TAIL_CALL_UNKNOWN_OPCODE, [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index d3eb02d47273474..9314fccc6c915a4 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -74,6 +74,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Set(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Slots(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_Sys(mod) < 0) { return NULL; } @@ -95,5 +98,8 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_File(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Weakref(mod) < 0) { + return NULL; + } return mod; } diff --git a/Modules/_testlimitedcapi/clinic/long.c.h b/Modules/_testlimitedcapi/clinic/long.c.h index ebaeb53921a82f1..f9852aba266a570 100644 --- a/Modules/_testlimitedcapi/clinic/long.c.h +++ b/Modules/_testlimitedcapi/clinic/long.c.h @@ -84,8 +84,8 @@ PyDoc_STRVAR(_testlimitedcapi_test_long_as_size_t__doc__, "\n" "Test the PyLong_As{Size,Ssize}_t API.\n" "\n" -"At present this just tests that non-integer arguments are handled correctly.\n" -"It should be extended to test overflow handling."); +"At present this just tests that non-integer arguments are handled\n" +"correctly. It should be extended to test overflow handling."); #define _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ {"test_long_as_size_t", (PyCFunction)_testlimitedcapi_test_long_as_size_t, METH_NOARGS, _testlimitedcapi_test_long_as_size_t__doc__}, @@ -140,4 +140,4 @@ PyDoc_STRVAR(_testlimitedcapi_PyLong_AsInt__doc__, #define _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF \ {"PyLong_AsInt", (PyCFunction)_testlimitedcapi_PyLong_AsInt, METH_O, _testlimitedcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=bc52b73c599f96c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb5c95bd0a4bdad8 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c index 34bc7331da92477..99b9e96760d50df 100644 --- a/Modules/_testlimitedcapi/long.c +++ b/Modules/_testlimitedcapi/long.c @@ -451,13 +451,13 @@ _testlimitedcapi.test_long_as_size_t Test the PyLong_As{Size,Ssize}_t API. -At present this just tests that non-integer arguments are handled correctly. -It should be extended to test overflow handling. +At present this just tests that non-integer arguments are handled +correctly. It should be extended to test overflow handling. [clinic start generated code]*/ static PyObject * _testlimitedcapi_test_long_as_size_t_impl(PyObject *module) -/*[clinic end generated code: output=297a9f14a42f55af input=8923d8f2038c46f4]*/ +/*[clinic end generated code: output=297a9f14a42f55af input=692e73744b35bf6e]*/ { size_t out_u; Py_ssize_t out_s; diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 1cbb4f5659af0f5..c51d285e19ab0d6 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -37,6 +37,7 @@ int _PyTestLimitedCAPI_Init_List(PyObject *module); int _PyTestLimitedCAPI_Init_Long(PyObject *module); int _PyTestLimitedCAPI_Init_PyOS(PyObject *module); int _PyTestLimitedCAPI_Init_Set(PyObject *module); +int _PyTestLimitedCAPI_Init_Slots(PyObject *module); int _PyTestLimitedCAPI_Init_Sys(PyObject *module); int _PyTestLimitedCAPI_Init_ThreadState(PyObject *module); int _PyTestLimitedCAPI_Init_Tuple(PyObject *module); @@ -44,5 +45,6 @@ int _PyTestLimitedCAPI_Init_Unicode(PyObject *module); int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); int _PyTestLimitedCAPI_Init_Version(PyObject *module); int _PyTestLimitedCAPI_Init_File(PyObject *module); +int _PyTestLimitedCAPI_Init_Weakref(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testlimitedcapi/slots.c b/Modules/_testlimitedcapi/slots.c new file mode 100644 index 000000000000000..9abe53d21154645 --- /dev/null +++ b/Modules/_testlimitedcapi/slots.c @@ -0,0 +1,672 @@ +#define Py_LIMITED_API 0x030f0000 + +#include "parts.h" + +PyABIInfo_VAR(abi_info); + +/* Define a bunch of (mostly nonsensical) functions to put in slots, so + * Lib/test/test_capi/test_slots.py can verify they've been assigned to + * the right slots. + + * This module is full of "magic constants" which simply need to match + * between the C and Python part of the tests. + */ + +// getbufferproc: export buffer; increment a counter +static int +demo_getbuffer(PyObject *exporter, Py_buffer *view, int flags) +{ + Py_INCREF(exporter); + // PyObject_GetTypeData & Py_TYPE: safe on non-subclassable type + int *data = PyObject_GetTypeData(exporter, Py_TYPE(exporter)); + if (!data) { + return -1; + } + (*data)++; + return PyBuffer_FillInfo(view, exporter, "buf", 4, 1, flags); +} + +// releasebufferproc: release buffer; decrement a counter +static void +demo_releasebuffer(PyObject *exporter, Py_buffer *view) +{ + Py_DECREF(exporter); + // PyObject_GetTypeData & Py_TYPE: safe on non-subclassable type + int *data = PyObject_GetTypeData(exporter, Py_TYPE(exporter)); + if (!data) { + PyErr_WriteUnraisable(exporter); + return; + } + (*data)--; + return; +} + +// objobjargproc: raise KeyError +static int +demo_ass_subscript(PyObject *o, PyObject *key, PyObject *v) +{ + PyErr_Format(PyExc_KeyError, "I don't like that key"); + return -1; +} + +// lenfunc: report 456 +static Py_ssize_t +demo_length(PyObject *o) +{ + return (Py_ssize_t)456; +} + +// binaryfunc; return constant value +static PyObject *binop_123(PyObject* a, PyObject *b) { return PyLong_FromLong(123); } +static PyObject *binop_234(PyObject* a, PyObject *b) { return PyLong_FromLong(234); } +static PyObject *binop_345(PyObject* a, PyObject *b) { return PyLong_FromLong(345); } +static PyObject *binop_456(PyObject* a, PyObject *b) { return PyLong_FromLong(456); } +static PyObject *binop_567(PyObject* a, PyObject *b) { return PyLong_FromLong(567); } +static PyObject *binop_678(PyObject* a, PyObject *b) { return PyLong_FromLong(678); } + +static PyObject * +type_from_slots(PyObject* module, PyObject *args) +{ + char *case_name; + if (!PyArg_ParseTuple(args, "s", &case_name)) { + return NULL; + } +#define CASE(NAME) \ + if (strcmp(case_name, NAME) == 0) { \ + return PyType_FromSlots((PySlot[]) { \ + PySlot_DATA(Py_tp_name, "_testlimitedcapi.MyType"), \ + PySlot_DATA(Py_tp_module, module), \ + ///////////////////////////////////////////////////////////////////////// +#define ENDCASE() \ + PySlot_END \ + }); \ + } \ + ///////////////////////////////////////////////////////////////////////// + + CASE("basic") + ENDCASE() + CASE("foreign_slot") + PySlot_DATA(Py_mod_name, "this is not a module"), + ENDCASE() + CASE("basicsize") + PySlot_SIZE(Py_tp_basicsize, 256), + ENDCASE() + CASE("extra_basicsize") + PySlot_SIZE(Py_tp_extra_basicsize, 256), + ENDCASE() + CASE("itemsize") + PySlot_SIZE(Py_tp_itemsize, 16), + ENDCASE() + CASE("flags") + PySlot_UINT64(Py_tp_flags, + Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_BASETYPE), + ENDCASE() + CASE("matmul_123") + PySlot_FUNC(Py_nb_matrix_multiply, binop_123), + ENDCASE() + CASE("optional_end") + {.sl_flags=PySlot_OPTIONAL}, + ENDCASE() + CASE("invalid") + {.sl_id=Py_slot_invalid}, + ENDCASE() + CASE("invalid_fbad") + {.sl_id=0xfbad}, + ENDCASE() + CASE("optional_invalid") + {.sl_id=Py_slot_invalid, .sl_flags=PySlot_OPTIONAL}, + PySlot_SIZE(Py_tp_extra_basicsize, 256), + ENDCASE() + CASE("optional_invalid_fbad") + {.sl_id=0xfbad, .sl_flags=PySlot_OPTIONAL}, + PySlot_SIZE(Py_tp_extra_basicsize, 256), + ENDCASE() + CASE("old_slot_numbers") + PySlot_FUNC(1, demo_getbuffer), + PySlot_FUNC(2, demo_releasebuffer), + PySlot_FUNC(3, demo_ass_subscript), + PySlot_FUNC(4, demo_length), + PySlot_SIZE(Py_tp_extra_basicsize, sizeof(int)), + PySlot_STATIC_DATA(Py_tp_members, ((PyMemberDef[]) { + {"buf_counter", Py_T_INT, 0, Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL}, + })), + ENDCASE() + CASE("new_slot_numbers") + PySlot_FUNC(88, demo_getbuffer), + PySlot_FUNC(89, demo_releasebuffer), + PySlot_FUNC(90, demo_ass_subscript), + PySlot_FUNC(91, demo_length), + PySlot_SIZE(Py_tp_extra_basicsize, sizeof(int)), + PySlot_STATIC_DATA(Py_tp_members, ((PyMemberDef[]) { + {"buf_counter", Py_T_INT, 0, Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL}, + })), + ENDCASE() + CASE("nonstatic_tp_members") + PySlot_SIZE(Py_tp_extra_basicsize, sizeof(int)), + PySlot_DATA(Py_tp_members, ((PyMemberDef[]) { + {"buf_counter", Py_T_INT, 0, Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL}, + })), + ENDCASE() + CASE("intptr_flags_macro") + PySlot_PTR(Py_tp_flags, (void*)(intptr_t)Py_TPFLAGS_IMMUTABLETYPE), + ENDCASE() + CASE("intptr_flags_struct") + {.sl_id=Py_tp_flags, + .sl_flags=PySlot_INTPTR, + .sl_ptr=(void*)(intptr_t)Py_TPFLAGS_IMMUTABLETYPE, + }, + ENDCASE() + CASE("intptr_static") + PySlot_SIZE(Py_tp_extra_basicsize, sizeof(int)), + PySlot_PTR_STATIC(Py_tp_members, ((PyMemberDef[]) { + {"attribute", Py_T_INT, 0, Py_READONLY | Py_RELATIVE_OFFSET}, + {NULL}, + })), + ENDCASE() + CASE("nested") + PySlot_FUNC(Py_nb_add, binop_123), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_subtract, binop_234), + PySlot_END, + })), + ENDCASE() + CASE("nested_max") + PySlot_FUNC(Py_nb_add, binop_123), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_subtract, binop_234), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_multiply, binop_345), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_true_divide, binop_456), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_remainder, binop_567), + PySlot_END + })), + PySlot_END, + })), + PySlot_END, + })), + PySlot_END, + })), + ENDCASE() + CASE("nested_over_limit") + PySlot_FUNC(Py_nb_add, binop_123), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_subtract, binop_234), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_multiply, binop_345), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_true_divide, binop_456), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_remainder, binop_567), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_xor, binop_678), + PySlot_END + })), + PySlot_END + })), + PySlot_END, + })), + PySlot_END, + })), + PySlot_END, + })), + ENDCASE() + CASE("nested_old") + PySlot_FUNC(Py_nb_add, binop_123), + PySlot_DATA(Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_subtract, binop_234}, + {0}, + })), + ENDCASE() + CASE("nested_old_max") + PySlot_FUNC(Py_nb_add, binop_123), + PySlot_DATA(Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_subtract, binop_234}, + {Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_multiply, binop_345}, + {Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_true_divide, binop_456}, + {Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_remainder, binop_567}, + {0}, + })}, + {0}, + })}, + {0}, + })}, + {0}, + })), + ENDCASE() + CASE("nested_old_over_limit") + PySlot_FUNC(Py_nb_add, binop_123), + PySlot_DATA(Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_subtract, binop_234}, + {Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_multiply, binop_345}, + {Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_true_divide, binop_456}, + {Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_remainder, binop_567}, + {Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_xor, binop_678}, + {0}, + })}, + {0}, + })}, + {0}, + })}, + {0}, + })}, + {0}, + })), + ENDCASE() + CASE("nested_pingpong") + PySlot_FUNC(Py_nb_add, binop_123), + PySlot_DATA(Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_subtract, binop_234}, + {Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_multiply, binop_345), + PySlot_DATA(Py_tp_slots, ((PyType_Slot[]) { + {Py_nb_true_divide, binop_456}, + {Py_slot_subslots, ((PySlot[]) { + PySlot_FUNC(Py_nb_remainder, binop_567), + PySlot_END + })}, + {0}, + })), + PySlot_END, + })}, + {0}, + })), + ENDCASE() + CASE("repeat_add") + PySlot_FUNC(Py_nb_add, binop_123), + PySlot_FUNC(Py_nb_add, binop_456), + ENDCASE() + CASE("repeat_module") + PySlot_DATA(Py_tp_module, Py_True), + PySlot_DATA(Py_tp_module, Py_False), + ENDCASE() + +#undef CASE +#undef ENDCASE + PyErr_Format(PyExc_SystemError, "bad case: %s", case_name); + return NULL; +} + +static PyObject * +type_from_null_slot(PyObject* module, PyObject *args) +{ + long slot_number; + if (!PyArg_ParseTuple(args, "l", &slot_number)) { + return NULL; + } + return PyType_FromSlots((PySlot[]) { + PySlot_DATA(Py_tp_name, "_testlimitedcapi.MyType"), + PySlot_DATA(Py_tp_module, module), + PySlot_PTR_STATIC((uint16_t)slot_number, NULL), + PySlot_END + }); +} + +static PyObject * +type_from_null_spec_slot(PyObject* Py_UNUSED(module), PyObject *args) +{ + long slot_number; + if (!PyArg_ParseTuple(args, "l", &slot_number)) { + return NULL; + } + return PyType_FromSpec(&(PyType_Spec) { + .name = "_testlimitedcapi.MyType", + .slots = (PyType_Slot[]) { + {slot_number, NULL}, + {0}, + }, + }); +} + +static PyObject * +demo_create(PyObject *spec, PyModuleDef *def) +{ + assert(def == NULL); + return Py_NewRef(spec); +} + +static int +demo_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "exec_done", "yes"); +} + +static PyMethodDef *TestMethods; + +static PyObject * +module_from_slots(PyObject* Py_UNUSED(module), PyObject *args) +{ + PyObject *spec; + char *case_name; + if (!PyArg_ParseTuple(args, "sO", &case_name, &spec)) { + return NULL; + } + PyObject *mod = NULL; +#define CASE(NAME) \ + if (strcmp(case_name, NAME) == 0) { \ + mod = PyModule_FromSlotsAndSpec((PySlot[]) { \ + PySlot_DATA(Py_mod_abi, &abi_info), \ + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), \ + ///////////////////////////////////////////////////////////////////////// +#define ENDCASE() \ + PySlot_END \ + }, spec); \ + } \ + ///////////////////////////////////////////////////////////////////////// + + CASE("basic") + ENDCASE() + CASE("foreign_slot") + PySlot_DATA(Py_tp_name, "this is not a type"), + ENDCASE() + CASE("state_size") + PySlot_SIZE(Py_mod_state_size, 42), + ENDCASE() + CASE("multi_interp") + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + ENDCASE() + CASE("exec") + PySlot_FUNC(Py_mod_exec, demo_exec), + ENDCASE() + CASE("optional_end") + {.sl_flags=PySlot_OPTIONAL}, + ENDCASE() + CASE("invalid") + {.sl_id=Py_slot_invalid}, + ENDCASE() + CASE("invalid_fbad") + {.sl_id=0xfbad}, + ENDCASE() + CASE("optional_invalid") + {.sl_id=Py_slot_invalid, .sl_flags=PySlot_OPTIONAL}, + PySlot_SIZE(Py_mod_exec, demo_exec), + ENDCASE() + CASE("optional_invalid_fbad") + {.sl_id=0xfbad, .sl_flags=PySlot_OPTIONAL}, + PySlot_SIZE(Py_mod_exec, demo_exec), + ENDCASE() + CASE("old_slot_numbers") + // 1: see old_slot_number_create case + PySlot_FUNC(2, demo_exec), + PySlot_DATA(3, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + // 4: see module_from_gil_slot function + ENDCASE() + CASE("new_slot_numbers") + // 84: see new_slot_number_create case + PySlot_FUNC(85, demo_exec), + PySlot_FUNC(86, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + // 87: see module_from_gil_slot function + ENDCASE() + CASE("old_slot_number_create") + PySlot_FUNC(1, demo_create), + ENDCASE() + CASE("new_slot_number_create") + PySlot_FUNC(84, demo_create), + ENDCASE() + CASE("nonstatic_mod_methods") + PySlot_DATA(Py_mod_methods, TestMethods), + ENDCASE() + CASE("intptr_methods") + PySlot_PTR_STATIC(Py_mod_methods, TestMethods), + ENDCASE() + CASE("nested") + PySlot_FUNC(Py_mod_exec, demo_exec), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_mod_doc, "doc"), + PySlot_END, + })), + ENDCASE() + CASE("nested_max") + PySlot_FUNC(Py_mod_exec, demo_exec), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_SIZE(Py_mod_state_size, 53), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_mod_doc, "doc"), + PySlot_END + })), + PySlot_END, + })), + PySlot_END, + })), + PySlot_END, + })), + ENDCASE() + CASE("nested_over_limit") + PySlot_FUNC(Py_mod_exec, demo_exec), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_SIZE(Py_mod_state_size, 53), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_mod_doc, "doc"), + PySlot_DATA(Py_slot_subslots, ((PySlot[]) { + PySlot_END + })), + PySlot_END + })), + PySlot_END, + })), + PySlot_END, + })), + PySlot_END, + })), + ENDCASE() + CASE("nested_old") + PySlot_FUNC(Py_mod_exec, demo_exec), + PySlot_DATA(Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_doc, "doc"}, + {0}, + })), + ENDCASE() + CASE("nested_old_max") + PySlot_FUNC(Py_mod_exec, demo_exec), + PySlot_DATA(Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_state_size, (void*)(intptr_t)53}, + {Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_doc, "doc"}, + {0}, + })}, + {0}, + })}, + {0}, + })}, + {0}, + })), + ENDCASE() + CASE("nested_old_over_limit") + PySlot_FUNC(Py_mod_exec, demo_exec), + PySlot_DATA(Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_state_size, (void*)(intptr_t)53}, + {Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_doc, "doc"}, + {0}, + })}, + {0}, + })}, + {0}, + })}, + {0}, + })}, + {0}, + })), + ENDCASE() + CASE("nested_pingpong") + PySlot_FUNC(Py_mod_exec, demo_exec), + PySlot_DATA(Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_mod_slots, ((PyModuleDef_Slot[]) { + {Py_mod_state_size, (void*)(intptr_t)53}, + {Py_slot_subslots, ((PySlot[]) { + PySlot_DATA(Py_mod_doc, "doc"), + PySlot_END + })}, + {0}, + })), + PySlot_END, + })}, + {0}, + })), + ENDCASE() + CASE("repeat_create") + PySlot_DATA(Py_mod_create, demo_create), + PySlot_DATA(Py_mod_create, demo_create), + PySlot_DATA(Py_mod_create, demo_create), + ENDCASE() + CASE("repeat_gil") + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + ENDCASE() + CASE("repeat_exec") + PySlot_FUNC(Py_mod_exec, demo_exec), + PySlot_FUNC(Py_mod_exec, demo_exec), + ENDCASE() + +#undef CASE +#undef ENDCASE + if (!mod) { + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_SystemError, "bad case: %s", case_name); + return NULL; + } + return NULL; + } + if (PyModule_Check(mod)) { + Py_ssize_t size; + if (PyModule_GetStateSize(mod, &size) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_AddIntConstant(mod, "state_size", (long)size) < 0) { + Py_DECREF(mod); + return NULL; + } + if (PyModule_Exec(mod) < 0) { + return NULL; + } + } + return mod; +} + +static PyObject * +module_from_gil_slot(PyObject* Py_UNUSED(module), PyObject *args) +{ + long slot_number; + PyObject *spec; + if (!PyArg_ParseTuple(args, "lO", &slot_number, &spec)) { + return NULL; + } + return PyModule_FromSlotsAndSpec((PySlot[]) { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_PTR_STATIC((uint16_t)slot_number, Py_MOD_GIL_NOT_USED), + PySlot_END + }, spec); +} + +static PyObject * +module_from_null_slot(PyObject* Py_UNUSED(module), PyObject *args) +{ + long slot_number; + PyObject *spec; + if (!PyArg_ParseTuple(args, "lO", &slot_number, &spec)) { + return NULL; + } + uint16_t maybe_gil_slot = Py_mod_gil; + if ((slot_number == 4) || (slot_number == 87)) { + // Do not repeat the GIL slot + maybe_gil_slot = Py_slot_invalid; + } + return PyModule_FromSlotsAndSpec((PySlot[]) { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_name, "mymod"), + PySlot_PTR_STATIC((uint16_t)slot_number, NULL), + { + .sl_id=maybe_gil_slot, + .sl_flags=PySlot_OPTIONAL, + .sl_ptr=Py_MOD_GIL_NOT_USED, + }, + PySlot_END + }, spec); +} + + + +static PyObject * +type_from_base_slots( + PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *base = NULL; + PyObject *bases = NULL; + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, "|OO", + (char*[]){"base", "bases", NULL}, + &base, &bases)) + { + return NULL; + } + + PySlot empty_slots[] = { + PySlot_END + }; + + PySlot base_slots[] = { + PySlot_DATA(Py_tp_base, base), + PySlot_END + }; + + PySlot bases_slots[] = { + PySlot_DATA(Py_tp_bases, bases), + PySlot_END + }; + + PySlot slots[] = { + PySlot_STATIC_DATA(Py_tp_name, "_testcapi.HeapCTypeWithBases"), + PySlot_UINT64(Py_tp_flags, Py_TPFLAGS_BASETYPE), + PySlot_DATA(Py_slot_subslots, base ? base_slots: empty_slots), + PySlot_DATA(Py_slot_subslots, bases ? bases_slots: empty_slots), + PySlot_END + }; + + return PyType_FromSlots(slots); +} + +static PyMethodDef _TestMethods[] = { + {"type_from_slots", type_from_slots, METH_VARARGS}, + {"module_from_gil_slot", module_from_gil_slot, METH_VARARGS}, + {"type_from_null_slot", type_from_null_slot, METH_VARARGS}, + {"type_from_null_spec_slot", type_from_null_spec_slot, METH_VARARGS}, + {"module_from_slots", module_from_slots, METH_VARARGS}, + {"module_from_null_slot", module_from_null_slot, METH_VARARGS}, + {"type_from_base_slots", _PyCFunction_CAST(type_from_base_slots), + METH_VARARGS | METH_KEYWORDS}, + {NULL}, +}; +static PyMethodDef *TestMethods = _TestMethods; + +int +_PyTestLimitedCAPI_Init_Slots(PyObject *m) +{ + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + return 0; +} diff --git a/Modules/_testlimitedcapi/weakref.c b/Modules/_testlimitedcapi/weakref.c new file mode 100644 index 000000000000000..e7f9d54d1a0d59e --- /dev/null +++ b/Modules/_testlimitedcapi/weakref.c @@ -0,0 +1,78 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.5 for PyModule_AddFunctions() +# define Py_LIMITED_API 0x03050000 +#endif + +#include "parts.h" +#include "util.h" + + +static PyObject * +pyweakref_check(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_Check(obj)); +} + +static PyObject * +pyweakref_checkref(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckRef(obj)); +} + +static PyObject * +pyweakref_checkrefexact(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckRefExact(obj)); +} + +static PyObject * +pyweakref_checkproxy(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckProxy(obj)); +} + +static PyObject * +pyweakref_newref(PyObject *module, PyObject *args) +{ + PyObject *obj; + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "O|O", &obj, &callback)) { + return NULL; + } + NULLABLE(obj); + return PyWeakref_NewRef(obj, callback); +} + +static PyObject * +pyweakref_newproxy(PyObject *module, PyObject *args) +{ + PyObject *obj; + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "O|O", &obj, &callback)) { + return NULL; + } + NULLABLE(obj); + return PyWeakref_NewProxy(obj, callback); +} + + +static PyMethodDef test_methods[] = { + {"pyweakref_check", pyweakref_check, METH_O}, + {"pyweakref_checkref", pyweakref_checkref, METH_O}, + {"pyweakref_checkrefexact", pyweakref_checkrefexact, METH_O}, + {"pyweakref_checkproxy", pyweakref_checkproxy, METH_O}, + {"pyweakref_newref", pyweakref_newref, METH_VARARGS}, + {"pyweakref_newproxy", pyweakref_newproxy, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Weakref(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index b0668e32eb57f4b..57ba61e67f1d7f4 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -141,14 +141,14 @@ _testmultiphase.StateAccessType.get_defining_module Return the module of the defining class. -Also tests that result of PyType_GetModuleByDef matches defining_class's -module. +Also tests that result of PyType_GetModuleByDef matches +defining_class's module. [clinic start generated code]*/ static PyObject * _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=ba2a14284a5d0921 input=d2c7245c8a9d06f8]*/ +/*[clinic end generated code: output=ba2a14284a5d0921 input=903e7f66555d65ae]*/ { PyObject *retval; retval = PyType_GetModule(cls); @@ -603,7 +603,7 @@ PyInit__testmultiphase_null_slots(void) /**** Problematic modules ****/ static PyModuleDef_Slot slots_bad_large[] = { - {_Py_mod_LAST_SLOT + 1, NULL}, + {Py_slot_invalid, NULL}, {0, NULL}, }; @@ -1051,12 +1051,12 @@ PyABIInfo_VAR(abi_info); PyMODEXPORT_FUNC PyModExport__test_from_modexport(void) { - static PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "_test_from_modexport"}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_name, "_test_from_modexport"), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return slots; } @@ -1064,12 +1064,12 @@ PyModExport__test_from_modexport(void) PyMODEXPORT_FUNC PyModExport__test_from_modexport_gil_used(void) { - static PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "_test_from_modexport_gil_used"}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_name, "_test_from_modexport_gil_used"), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_USED), + PySlot_END, }; return slots; } @@ -1115,13 +1115,13 @@ modexport_create_string(PyObject *spec, PyModuleDef *def) PyMODEXPORT_FUNC PyModExport__test_from_modexport_create_nonmodule(void) { - static PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "_test_from_modexport_create_nonmodule"}, - {Py_mod_create, modexport_create_string}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_name, "_test_from_modexport_create_nonmodule"), + PySlot_FUNC(Py_mod_create, modexport_create_string), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return slots; } @@ -1129,19 +1129,19 @@ PyModExport__test_from_modexport_create_nonmodule(void) PyMODEXPORT_FUNC PyModExport__test_from_modexport_create_nonmodule_gil_used(void) { - static PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "_test_from_modexport_create_nonmodule"}, - {Py_mod_create, modexport_create_string}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_name, "_test_from_modexport_create_nonmodule"), + PySlot_FUNC(Py_mod_create, modexport_create_string), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_USED), + PySlot_END, }; return slots; } -static PyModuleDef_Slot modexport_empty_slots[] = { - {0}, +static PySlot modexport_empty_slots[] = { + PySlot_END, }; PyMODEXPORT_FUNC @@ -1151,9 +1151,9 @@ PyModExport__test_from_modexport_empty_slots(void) } -static PyModuleDef_Slot modexport_minimal_slots[] = { - {Py_mod_abi, &abi_info}, - {0}, +static PySlot modexport_minimal_slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_END, }; PyMODEXPORT_FUNC @@ -1234,18 +1234,18 @@ PyModExport__test_from_modexport_smoke(void) {"get_modexport_minimal_slots", modexport_get_minimal_slots, METH_NOARGS}, {0}, }; - static PyModuleDef_Slot slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "_test_from_modexport_smoke"}, - {Py_mod_doc, "the expected docstring"}, - {Py_mod_exec, modexport_smoke_exec}, - {Py_mod_state_size, (void*)sizeof(int)}, - {Py_mod_methods, methods}, - {Py_mod_state_free, modexport_smoke_free}, - {Py_mod_token, (void*)&modexport_smoke_test_token}, - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - {0}, + static PySlot slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), + PySlot_DATA(Py_mod_name, "_test_from_modexport_smoke"), + PySlot_DATA(Py_mod_doc, "the expected docstring"), + PySlot_FUNC(Py_mod_exec, modexport_smoke_exec), + PySlot_SIZE(Py_mod_state_size, (void*)sizeof(int)), + PySlot_STATIC_DATA(Py_mod_methods, methods), + PySlot_FUNC(Py_mod_state_free, modexport_smoke_free), + PySlot_DATA(Py_mod_token, (void*)&modexport_smoke_test_token), + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_END, }; return slots; } diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 135b53111014d1c..7a34c5a5a0ef6be 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -820,8 +820,8 @@ _thread.lock.acquire Lock the lock. -Without argument, this blocks if the lock is already -locked (even by the same thread), waiting for another thread to release +Without argument, this blocks if the lock is already locked +(even by the same thread), waiting for another thread to release the lock, and return True once the lock is acquired. With an argument, this will only block if the argument is true, and the return value reflects whether the lock is acquired. @@ -831,7 +831,7 @@ The blocking operation is interruptible. static PyObject * _thread_lock_acquire_impl(lockobject *self, int blocking, PyObject *timeoutobj) -/*[clinic end generated code: output=569d6b25d508bf6f input=13e999649bc1c798]*/ +/*[clinic end generated code: output=569d6b25d508bf6f input=73e75b3d2ec32677]*/ { PyTime_t timeout; @@ -1131,19 +1131,19 @@ _thread.RLock.release Release the lock. -Allows another thread that is blocked waiting for -the lock to acquire the lock. The lock must be in the locked state, +Allows another thread that is blocked waiting for the lock +to acquire the lock. The lock must be in the locked state, and must be locked by the same thread that unlocks it; otherwise a `RuntimeError` is raised. -Do note that if the lock was acquire()d several times in a row by the -current thread, release() needs to be called as many times for the lock -to be available for other threads. +Do note that if the lock was acquire()d several times in a row by +the current thread, release() needs to be called as many times for +the lock to be available for other threads. [clinic start generated code]*/ static PyObject * _thread_RLock_release_impl(rlockobject *self) -/*[clinic end generated code: output=51f4a013c5fae2c5 input=d425daf1a5782e63]*/ +/*[clinic end generated code: output=51f4a013c5fae2c5 input=7c188f60189be13a]*/ { if (_PyRecursiveMutex_TryUnlock(&self->lock) < 0) { PyErr_SetString(PyExc_RuntimeError, diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index bbe2a428454e0cc..466e275c5cecf0e 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -53,6 +53,10 @@ Copyright (C) 1994 Steen Lumholt. # include <tk.h> #endif +#if defined(MS_WINDOWS) && TK_MAJOR_VERSION >= 9 +# include <tkPlatDecls.h> +#endif + #include "tkinter.h" #if TK_HEX_VERSION < 0x0805020c @@ -128,18 +132,20 @@ _get_tcl_lib_path(void) } /* Check expected location for an installed Python first */ - tcl_library_path = PyUnicode_FromString("\\tcl\\tcl" TCL_VERSION); - if (tcl_library_path == NULL) { + PyObject* tmp_tcl_library_path = PyUnicode_FromString("\\tcl\\tcl" TCL_VERSION); + if (tmp_tcl_library_path == NULL) { Py_DECREF(prefix); return NULL; } - tcl_library_path = PyUnicode_Concat(prefix, tcl_library_path); + tcl_library_path = PyUnicode_Concat(prefix, tmp_tcl_library_path); + Py_DECREF(tmp_tcl_library_path); Py_DECREF(prefix); if (tcl_library_path == NULL) { return NULL; } stat_return_value = _Py_stat(tcl_library_path, &stat_buf); if (stat_return_value == -2) { + Py_DECREF(tcl_library_path); return NULL; } if (stat_return_value == -1) { @@ -154,16 +160,17 @@ _get_tcl_lib_path(void) } stat_return_value = _Py_stat(tcl_library_path, &stat_buf); if (stat_return_value == -2) { + Py_DECREF(tcl_library_path); return NULL; } if (stat_return_value == -1) { /* tcltkDir for a repository build doesn't exist either, reset errno and leave Tcl to its own devices */ errno = 0; - tcl_library_path = NULL; + Py_CLEAR(tcl_library_path); } #else - tcl_library_path = NULL; + Py_CLEAR(tcl_library_path); #endif } already_checked = 1; @@ -172,6 +179,57 @@ _get_tcl_lib_path(void) } #endif /* MS_WINDOWS */ +#if defined(MS_WINDOWS) && TK_MAJOR_VERSION >= 9 +static void +mount_tk_dll_zip(void) +{ + HINSTANCE tk_module = Tk_GetHINSTANCE(); + wchar_t *tk_path = NULL; + DWORD path_len = 0; + for (DWORD buffer_len = 256; + tk_path == NULL && buffer_len < (1024 * 1024); + buffer_len *= 2) + { + tk_path = (wchar_t *)PyMem_RawMalloc( + buffer_len * sizeof(*tk_path)); + if (tk_path != NULL) { + path_len = GetModuleFileNameW(tk_module, tk_path, buffer_len); + if (path_len == buffer_len) { + PyMem_RawFree(tk_path); + tk_path = NULL; + } + } + } + + if (tk_path == NULL || path_len == 0) { + PyMem_RawFree(tk_path); + return; + } + + Tcl_DString utf8_path; + + Tcl_DStringInit(&utf8_path); + Tcl_WCharToUtfDString(tk_path, path_len, &utf8_path); + /* Failure is harmless if the DLL has no embedded ZIP or if another + interpreter has already mounted it. */ + (void) TclZipfs_Mount(NULL, Tcl_DStringValue(&utf8_path), + "//zipfs:/lib/tk", NULL); + Tcl_DStringFree(&utf8_path); + PyMem_RawFree(tk_path); +} +#endif + +int +Tkinter_TkInit(Tcl_Interp *interp) +{ +#if defined(MS_WINDOWS) && TK_MAJOR_VERSION >= 9 + /* Tcl/Tk 9 may embed the tk_library in the Tk DLL which tcl_findLibrary + does not search. Mount the DLL using Zipfs if possible. */ + mount_tk_dll_zip(); +#endif + return Tk_Init(interp); +} + /* The threading situation is complicated. Tcl is not thread-safe, except when configured with --enable-threads. @@ -541,7 +599,7 @@ Tcl_AppInit(Tcl_Interp *interp) return TCL_OK; } - if (Tk_Init(interp) == TCL_ERROR) { + if (Tkinter_TkInit(interp) == TCL_ERROR) { PySys_WriteStderr("Tk_Init error: %s\n", Tcl_GetStringResult(interp)); return TCL_ERROR; } @@ -707,11 +765,13 @@ Tkapp_New(const char *screenName, const char *className, if (!ret && GetLastError() == ERROR_ENVVAR_NOT_FOUND) { str_path = _get_tcl_lib_path(); if (str_path == NULL && PyErr_Occurred()) { + Py_DECREF(v); return NULL; } if (str_path != NULL) { utf8_path = PyUnicode_AsUTF8String(str_path); if (utf8_path == NULL) { + Py_DECREF(v); return NULL; } Tcl_SetVar(v->interp, @@ -2983,7 +3043,7 @@ _tkinter_tkapp_loadtk_impl(TkappObject *self) return NULL; } if (_tk_exists == NULL || strcmp(_tk_exists, "1") != 0) { - if (Tk_Init(interp) == TCL_ERROR) { + if (Tkinter_TkInit(interp) == TCL_ERROR) { Tkinter_Error(self); return NULL; } @@ -3212,6 +3272,20 @@ _tkinter_create_impl(PyObject *module, const char *screenName, CHECK_STRING_LENGTH(className); CHECK_STRING_LENGTH(use); +#if TCL_MAJOR_VERSION < 9 + /* className is title-cased during Tk initialization. Tcl 8.x does not + * support non-BMP characters (encoded as 4-byte UTF-8 sequences) there + * and crashes in Tcl_UtfToTitle (see gh-126219). Reject them up front. */ + for (const unsigned char *p = (const unsigned char *)className; *p; p++) { + if (*p >= 0xF0) { + PyErr_SetString(PyExc_ValueError, + "className must not contain non-BMP characters " + "with this version of Tcl/Tk"); + return NULL; + } + } +#endif + return (PyObject *) Tkapp_New(screenName, className, interactive, wantobjects, wantTk, sync, use); @@ -3219,7 +3293,6 @@ _tkinter_create_impl(PyObject *module, const char *screenName, /*[clinic input] @permit_long_summary -@permit_long_docstring_body _tkinter.setbusywaitinterval new_val: int @@ -3227,12 +3300,13 @@ _tkinter.setbusywaitinterval Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter. -It should be set to a divisor of the maximum time between frames in an animation. +It should be set to a divisor of the maximum time between frames in +an animation. [clinic start generated code]*/ static PyObject * _tkinter_setbusywaitinterval_impl(PyObject *module, int new_val) -/*[clinic end generated code: output=42bf7757dc2d0ab6 input=07b82a04b56625e1]*/ +/*[clinic end generated code: output=42bf7757dc2d0ab6 input=0360dd95c8bd8619]*/ { if (new_val < 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 56d83ea0dcb2a7d..a97b0e0bfeefba6 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -11,6 +11,7 @@ module _tracemalloc /*[clinic input] +@permit_long_summary _tracemalloc.is_tracing Return True if the tracemalloc module is tracing Python memory allocations. @@ -18,7 +19,7 @@ Return True if the tracemalloc module is tracing Python memory allocations. static PyObject * _tracemalloc_is_tracing_impl(PyObject *module) -/*[clinic end generated code: output=2d763b42601cd3ef input=af104b0a00192f63]*/ +/*[clinic end generated code: output=2d763b42601cd3ef input=cac4fc9096babeac]*/ { return PyBool_FromLong(_PyTraceMalloc_IsTracing()); } @@ -153,6 +154,7 @@ _tracemalloc_get_tracemalloc_memory_impl(PyObject *module) /*[clinic input] +@permit_long_summary _tracemalloc.get_traced_memory Get the current size and peak size of memory blocks traced by tracemalloc. @@ -162,7 +164,7 @@ Returns a tuple: (current: int, peak: int). static PyObject * _tracemalloc_get_traced_memory_impl(PyObject *module) -/*[clinic end generated code: output=5b167189adb9e782 input=61ddb5478400ff66]*/ +/*[clinic end generated code: output=5b167189adb9e782 input=b06e7a1a4914fc21]*/ { return _PyTraceMalloc_GetTracedMemory(); } diff --git a/Modules/_winapi.c b/Modules/_winapi.c index ffa407b2f21f733..535784adedb24d8 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -49,6 +49,13 @@ #include <crtdbg.h> #include "winreparse.h" +// PSAPI_VERSION=2 redirects GetProcessMemoryInfo() to +// K32GetProcessMemoryInfo() in kernel32.dll, so we don't need to link +// psapi.lib. See: +// https://learn.microsoft.com/windows/win32/api/psapi/nf-psapi-getprocessmemoryinfo +#define PSAPI_VERSION 2 +#include <psapi.h> // GetProcessMemoryInfo() + #if defined(MS_WIN32) && !defined(MS_WIN64) #define HANDLE_TO_PYNUM(handle) \ PyLong_FromUnsignedLong((unsigned long) handle) @@ -1187,8 +1194,10 @@ gethandlelist(PyObject *mapping, const char *name, Py_ssize_t *size) } ret = PyMem_Malloc(*size); - if (ret == NULL) + if (ret == NULL) { + PyErr_NoMemory(); goto cleanup; + } for (i = 0; i < PySequence_Fast_GET_SIZE(value_fast); i++) { ret[i] = PYNUM_TO_HANDLE(PySequence_Fast_GET_ITEM(value_fast, i)); @@ -1271,6 +1280,7 @@ getattributelist(PyObject *obj, const char *name, AttributeList *attribute_list) attribute_list->attribute_list = PyMem_Malloc(attribute_list_size); if (attribute_list->attribute_list == NULL) { ret = -1; + PyErr_NoMemory(); goto cleanup; } @@ -1674,6 +1684,9 @@ _winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path) } PyMem_Free((void *)buffer); } + else { + PyErr_NoMemory(); + } } else { PyErr_SetFromWindowsErr(0); } @@ -2384,6 +2397,7 @@ _winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, while (i < nhandles) { BatchedWaitData *data = (BatchedWaitData*)PyMem_Malloc(sizeof(BatchedWaitData)); if (!data) { + PyErr_NoMemory(); goto error; } thread_data[thread_count++] = data; @@ -3080,6 +3094,76 @@ _winapi_ReportEvent_impl(PyObject *module, HANDLE handle, } +/*[clinic input] +_winapi.GetProcessMemoryInfo + handle: HANDLE + / + +Return the memory usage of the given process handle as a dict. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetProcessMemoryInfo_impl(PyObject *module, HANDLE handle) +/*[clinic end generated code: output=00a5d09732e84120 input=5b90ad61cdc68d2a]*/ +{ + PROCESS_MEMORY_COUNTERS pmc; + if (!GetProcessMemoryInfo(handle, &pmc, sizeof(pmc))) { + return PyErr_SetFromWindowsErr(0); + } + + PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + +#define ADD(ATTR) \ + do { \ + PyObject *obj = PyLong_FromSize_t(pmc.ATTR); \ + if (obj == NULL) { \ + goto error; \ + } \ + if (PyDict_SetItemString(result, #ATTR, obj) < 0) { \ + Py_DECREF(obj); \ + goto error; \ + } \ + Py_DECREF(obj); \ + } while (0) + + ADD(PageFaultCount); + ADD(PeakWorkingSetSize); + ADD(WorkingSetSize); + ADD(QuotaPeakPagedPoolUsage); + ADD(QuotaPagedPoolUsage); + ADD(QuotaPeakNonPagedPoolUsage); + ADD(QuotaNonPagedPoolUsage); + ADD(PagefileUsage); + ADD(PeakPagefileUsage); + +#undef ADD + + return result; + +error: + Py_DECREF(result); + return NULL; +} + + +/*[clinic input] +_winapi.GetTickCount64 + +Number of milliseconds that have elapsed since the system was started. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetTickCount64_impl(PyObject *module) +/*[clinic end generated code: output=cb33c0568f0b3ed1 input=77ed6539ac7d6590]*/ +{ + ULONGLONG ticks = GetTickCount64(); + return PyLong_FromUnsignedLongLong(ticks); +} + + static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF _WINAPI_CONNECTNAMEDPIPE_METHODDEF @@ -3130,6 +3214,8 @@ static PyMethodDef winapi_functions[] = { _WINAPI__MIMETYPES_READ_WINDOWS_REGISTRY_METHODDEF _WINAPI_NEEDCURRENTDIRECTORYFOREXEPATH_METHODDEF _WINAPI_COPYFILE2_METHODDEF + _WINAPI_GETPROCESSMEMORYINFO_METHODDEF + _WINAPI_GETTICKCOUNT64_METHODDEF {NULL, NULL} }; @@ -3226,6 +3312,7 @@ static int winapi_exec(PyObject *m) WINAPI_CONSTANT(F_DWORD, PROCESS_ALL_ACCESS); WINAPI_CONSTANT(F_DWORD, SYNCHRONIZE); WINAPI_CONSTANT(F_DWORD, PROCESS_DUP_HANDLE); + WINAPI_CONSTANT(F_DWORD, PROCESS_QUERY_LIMITED_INFORMATION); WINAPI_CONSTANT(F_DWORD, SEC_COMMIT); WINAPI_CONSTANT(F_DWORD, SEC_IMAGE); WINAPI_CONSTANT(F_DWORD, SEC_LARGE_PAGES); diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index 4f8d93fd9e867c6..6775ba4826a652b 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -21,8 +21,8 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__, " zstd_dict\n" " A ZstdDict object, a pre-trained Zstandard dictionary.\n" "\n" -"Thread-safe at method level. For one-shot compression, use the compress()\n" -"function instead."); +"Thread-safe at method level. For one-shot compression, use the\n" +"compress() function instead."); static PyObject * _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, @@ -105,9 +105,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_compress__doc__, " Can be these 3 values ZstdCompressor.CONTINUE,\n" " ZstdCompressor.FLUSH_BLOCK, ZstdCompressor.FLUSH_FRAME\n" "\n" -"Return a chunk of compressed data if possible, or b\'\' otherwise. When you have\n" -"finished providing data to the compressor, call the flush() method to finish\n" -"the compression process."); +"Return a chunk of compressed data if possible, or b\'\' otherwise.\n" +"When you have finished providing data to the compressor, call the\n" +"flush() method to finish the compression process."); #define _ZSTD_ZSTDCOMPRESSOR_COMPRESS_METHODDEF \ {"compress", _PyCFunction_CAST(_zstd_ZstdCompressor_compress), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_compress__doc__}, @@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__, " Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n" " ZstdCompressor.FLUSH_BLOCK\n" "\n" -"Flush any remaining data left in internal buffers. Since Zstandard data\n" -"consists of one or more independent frames, the compressor object can still\n" -"be used after this method is called."); +"Flush any remaining data left in internal buffers. Since Zstandard\n" +"data consists of one or more independent frames, the compressor\n" +"object can still be used after this method is called."); #define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \ {"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__}, @@ -262,13 +262,14 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_set_pledged_input_size__doc__, " size\n" " The size of the uncompressed data to be provided to the compressor.\n" "\n" -"This method can be used to ensure the header of the frame about to be written\n" -"includes the size of the data, unless the CompressionParameter.content_size_flag\n" -"is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n" +"This method can be used to ensure the header of the frame about to\n" +"be written includes the size of the data, unless the\n" +"CompressionParameter.content_size_flag is set to False.\n" +"If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n" "\n" -"It is important to ensure that the pledged data size matches the actual data\n" -"size. If they do not match the compressed output data may be corrupted and the\n" -"final chunk written may be lost."); +"It is important to ensure that the pledged data size matches the\n" +"actual data size. If they do not match the compressed output data\n" +"may be corrupted and the final chunk written may be lost."); #define _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF \ {"set_pledged_input_size", (PyCFunction)_zstd_ZstdCompressor_set_pledged_input_size, METH_O, _zstd_ZstdCompressor_set_pledged_input_size__doc__}, @@ -291,4 +292,4 @@ _zstd_ZstdCompressor_set_pledged_input_size(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=c1d5c2cf06a8becd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1a5e21476885866c input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index c6fdae74ab0447b..fe3b76b8bb369df 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -20,8 +20,8 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__, " options\n" " A dict object that contains advanced decompression parameters.\n" "\n" -"Thread-safe at method level. For one-shot decompression, use the decompress()\n" -"function instead."); +"Thread-safe at method level. For one-shot decompression, use the\n" +"decompress() function instead."); static PyObject * _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, @@ -91,7 +91,8 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_unused_data__doc__, "A bytes object of un-consumed input data.\n" "\n" "When ZstdDecompressor object stops after a frame is\n" -"decompressed, unused input data after the frame. Otherwise this will be b\'\'."); +"decompressed, unused input data after the frame. Otherwise this\n" +"will be b\'\'."); #if defined(_zstd_ZstdDecompressor_unused_data_DOCSTR) # undef _zstd_ZstdDecompressor_unused_data_DOCSTR #endif @@ -129,18 +130,19 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, " output buffer is unlimited. When it is nonnegative, returns at\n" " most max_length bytes of decompressed data.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of a frame is reached raises an\n" -"EOFError. Any data found after the end of the frame is ignored and saved in\n" -"the self.unused_data attribute."); +"Attempting to decompress data after the end of a frame is reached\n" +"raises an EOFError. Any data found after the end of the frame is\n" +"ignored and saved in the self.unused_data attribute."); #define _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_zstd_ZstdDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdDecompressor_decompress__doc__}, @@ -220,4 +222,4 @@ _zstd_ZstdDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=30c12ef047027ede input=a9049054013a1b77]*/ +/*[clinic end generated code: output=70bc308e86463751 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 166d925a542352d..18b049e3cbe37ef 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -21,8 +21,8 @@ PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, " advanced cases. Otherwise, check that the content represents\n" " a Zstandard dictionary created by the zstd library or CLI.\n" "\n" -"The dictionary can be used for compression or decompression, and can be shared\n" -"by multiple ZstdCompressor or ZstdDecompressor objects."); +"The dictionary can be used for compression or decompression, and can be\n" +"shared by multiple ZstdCompressor or ZstdDecompressor objects."); static PyObject * _zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, @@ -125,11 +125,11 @@ PyDoc_STRVAR(_zstd_ZstdDict_as_digested_dict__doc__, "Pass this attribute as zstd_dict argument:\n" "compress(dat, zstd_dict=zd.as_digested_dict)\n" "\n" -"1. Some advanced compression parameters of compressor may be overridden\n" -" by parameters of digested dictionary.\n" -"2. ZstdDict has a digested dictionaries cache for each compression level.\n" -" It\'s faster when loading again a digested dictionary with the same\n" -" compression level.\n" +"1. Some advanced compression parameters of compressor may be\n" +" overridden by parameters of digested dictionary.\n" +"2. ZstdDict has a digested dictionaries cache for each compression\n" +" level. It\'s faster when loading again a digested dictionary with\n" +" the same compression level.\n" "3. No need to use this for decompression."); #if defined(_zstd_ZstdDict_as_digested_dict_DOCSTR) # undef _zstd_ZstdDict_as_digested_dict_DOCSTR @@ -161,9 +161,10 @@ PyDoc_STRVAR(_zstd_ZstdDict_as_undigested_dict__doc__, "Pass this attribute as zstd_dict argument:\n" "compress(dat, zstd_dict=zd.as_undigested_dict)\n" "\n" -"1. The advanced compression parameters of compressor will not be overridden.\n" -"2. Loading an undigested dictionary is costly. If load an undigested dictionary\n" -" multiple times, consider reusing a compressor object.\n" +"1. The advanced compression parameters of compressor will not be\n" +" overridden.\n" +"2. Loading an undigested dictionary is costly. If load an undigested\n" +" dictionary multiple times, consider reusing a compressor object.\n" "3. No need to use this for decompression."); #if defined(_zstd_ZstdDict_as_undigested_dict_DOCSTR) # undef _zstd_ZstdDict_as_undigested_dict_DOCSTR @@ -195,9 +196,10 @@ PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__, "Pass this attribute as zstd_dict argument:\n" "compress(dat, zstd_dict=zd.as_prefix)\n" "\n" -"1. Prefix is compatible with long distance matching, while dictionary is not.\n" -"2. It only works for the first frame, then the compressor/decompressor will\n" -" return to no prefix state.\n" +"1. Prefix is compatible with long distance matching, while\n" +" dictionary is not.\n" +"2. It only works for the first frame, then the\n" +" compressor/decompressor will return to no prefix state.\n" "3. When decompressing, must use the same prefix as when compressing."); #if defined(_zstd_ZstdDict_as_prefix_DOCSTR) # undef _zstd_ZstdDict_as_prefix_DOCSTR @@ -222,4 +224,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) { return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); } -/*[clinic end generated code: output=f41d9e2e2cc2928f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=49b66061b4fcdb5f input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 8a3cd182ab15160..b2eb22d9ec8add8 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -332,14 +332,14 @@ _zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new Create a compressor object for compressing data incrementally. -Thread-safe at method level. For one-shot compression, use the compress() -function instead. +Thread-safe at method level. For one-shot compression, use the +compress() function instead. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, PyObject *options, PyObject *zstd_dict) -/*[clinic end generated code: output=cdef61eafecac3d7 input=92de0211ae20ffdc]*/ +/*[clinic end generated code: output=cdef61eafecac3d7 input=bbfeeaa06fd3bd4d]*/ { ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { @@ -583,7 +583,6 @@ compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data) } /*[clinic input] -@permit_long_docstring_body _zstd.ZstdCompressor.compress data: Py_buffer @@ -593,15 +592,15 @@ _zstd.ZstdCompressor.compress Provide data to the compressor object. -Return a chunk of compressed data if possible, or b'' otherwise. When you have -finished providing data to the compressor, call the flush() method to finish -the compression process. +Return a chunk of compressed data if possible, or b'' otherwise. +When you have finished providing data to the compressor, call the +flush() method to finish the compression process. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, int mode) -/*[clinic end generated code: output=ed7982d1cf7b4f98 input=6018ed6cc729cea6]*/ +/*[clinic end generated code: output=ed7982d1cf7b4f98 input=11726dff64d7b2f9]*/ { PyObject *ret; @@ -643,7 +642,6 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, } /*[clinic input] -@permit_long_docstring_body _zstd.ZstdCompressor.flush mode: int(c_default="ZSTD_e_end") = ZstdCompressor.FLUSH_FRAME @@ -652,14 +650,14 @@ _zstd.ZstdCompressor.flush Finish the compression process. -Flush any remaining data left in internal buffers. Since Zstandard data -consists of one or more independent frames, the compressor object can still -be used after this method is called. +Flush any remaining data left in internal buffers. Since Zstandard +data consists of one or more independent frames, the compressor +object can still be used after this method is called. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) -/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a9871ec742d79003]*/ +/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=130e0b1eddf0f498]*/ { PyObject *ret; @@ -693,7 +691,7 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) /*[clinic input] -@permit_long_docstring_body +@permit_long_summary _zstd.ZstdCompressor.set_pledged_input_size size: zstd_contentsize @@ -702,19 +700,20 @@ _zstd.ZstdCompressor.set_pledged_input_size Set the uncompressed content size to be written into the frame header. -This method can be used to ensure the header of the frame about to be written -includes the size of the data, unless the CompressionParameter.content_size_flag -is set to False. If last_mode != FLUSH_FRAME, then a RuntimeError is raised. +This method can be used to ensure the header of the frame about to +be written includes the size of the data, unless the +CompressionParameter.content_size_flag is set to False. +If last_mode != FLUSH_FRAME, then a RuntimeError is raised. -It is important to ensure that the pledged data size matches the actual data -size. If they do not match the compressed output data may be corrupted and the -final chunk written may be lost. +It is important to ensure that the pledged data size matches the +actual data size. If they do not match the compressed output data +may be corrupted and the final chunk written may be lost. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self, unsigned long long size) -/*[clinic end generated code: output=3a09e55cc0e3b4f9 input=b4c87bcbd5ce6111]*/ +/*[clinic end generated code: output=3a09e55cc0e3b4f9 input=714cd7a9aa10e2a8]*/ { // Error occurred while converting argument, should be unreachable assert(size != ZSTD_CONTENTSIZE_ERROR); diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 46682b483ad06ab..cb95ba89eb650ae 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -469,7 +469,6 @@ stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data, /*[clinic input] -@permit_long_docstring_body @classmethod _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new zstd_dict: object = None @@ -479,14 +478,14 @@ _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new Create a decompressor object for decompressing data incrementally. -Thread-safe at method level. For one-shot decompression, use the decompress() -function instead. +Thread-safe at method level. For one-shot decompression, use the +decompress() function instead. [clinic start generated code]*/ static PyObject * _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, PyObject *options) -/*[clinic end generated code: output=590ca65c1102ff4a input=ed8891edfd14cdaa]*/ +/*[clinic end generated code: output=590ca65c1102ff4a input=73879de69bf89f59]*/ { ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { @@ -571,19 +570,19 @@ ZstdDecompressor_dealloc(PyObject *ob) } /*[clinic input] -@permit_long_docstring_body @getter _zstd.ZstdDecompressor.unused_data A bytes object of un-consumed input data. When ZstdDecompressor object stops after a frame is -decompressed, unused input data after the frame. Otherwise this will be b''. +decompressed, unused input data after the frame. Otherwise this +will be b''. [clinic start generated code]*/ static PyObject * _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) -/*[clinic end generated code: output=f3a20940f11b6b09 input=37c2c531ab56f914]*/ +/*[clinic end generated code: output=f3a20940f11b6b09 input=0462065c5e60ba01]*/ { PyObject *ret; @@ -613,7 +612,6 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) /*[clinic input] @permit_long_summary -@permit_long_docstring_body _zstd.ZstdDecompressor.decompress data: Py_buffer @@ -625,25 +623,26 @@ _zstd.ZstdDecompressor.decompress Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of a frame is reached raises an -EOFError. Any data found after the end of the frame is ignored and saved in -the self.unused_data attribute. +Attempting to decompress data after the end of a frame is reached +raises an EOFError. Any data found after the end of the frame is +ignored and saved in the self.unused_data attribute. [clinic start generated code]*/ static PyObject * _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=a4302b3c940dbec6 input=e5c905a774df1553]*/ +/*[clinic end generated code: output=a4302b3c940dbec6 input=4ddda5a0bdd00673]*/ { PyObject *ret; /* Thread-safe code */ diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index b0bfbdc886e04fa..e1b9d998e697fb8 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -23,7 +23,6 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" #define ZstdDict_CAST(op) ((ZstdDict *)op) /*[clinic input] -@permit_long_docstring_body @classmethod _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new dict_content: Py_buffer @@ -37,14 +36,14 @@ _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new Represents a Zstandard dictionary. -The dictionary can be used for compression or decompression, and can be shared -by multiple ZstdCompressor or ZstdDecompressor objects. +The dictionary can be used for compression or decompression, and can be +shared by multiple ZstdCompressor or ZstdDecompressor objects. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, int is_raw) -/*[clinic end generated code: output=685b7406a48b0949 input=b132ee40b784c293]*/ +/*[clinic end generated code: output=685b7406a48b0949 input=3bb66063c0240433]*/ { /* All dictionaries must be at least 8 bytes */ if (dict_content->len < 8) { @@ -154,7 +153,6 @@ _zstd_ZstdDict_dict_content_get_impl(ZstdDict *self) } /*[clinic input] -@permit_long_docstring_body @getter _zstd.ZstdDict.as_digested_dict @@ -163,23 +161,22 @@ Load as a digested dictionary to compressor. Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict) -1. Some advanced compression parameters of compressor may be overridden - by parameters of digested dictionary. -2. ZstdDict has a digested dictionaries cache for each compression level. - It's faster when loading again a digested dictionary with the same - compression level. +1. Some advanced compression parameters of compressor may be + overridden by parameters of digested dictionary. +2. ZstdDict has a digested dictionaries cache for each compression + level. It's faster when loading again a digested dictionary with + the same compression level. 3. No need to use this for decompression. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=09b086e7a7320dbb input=8d01ff0b8b043f2e]*/ +/*[clinic end generated code: output=09b086e7a7320dbb input=a9417d40f1d7fedd]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED); } /*[clinic input] -@permit_long_docstring_body @getter _zstd.ZstdDict.as_undigested_dict @@ -188,21 +185,21 @@ Load as an undigested dictionary to compressor. Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict) -1. The advanced compression parameters of compressor will not be overridden. -2. Loading an undigested dictionary is costly. If load an undigested dictionary - multiple times, consider reusing a compressor object. +1. The advanced compression parameters of compressor will not be + overridden. +2. Loading an undigested dictionary is costly. If load an undigested + dictionary multiple times, consider reusing a compressor object. 3. No need to use this for decompression. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=43c7a989e6d4253a input=b1bdb306c3798ad4]*/ +/*[clinic end generated code: output=43c7a989e6d4253a input=56443c9c4e589cd5]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED); } /*[clinic input] -@permit_long_docstring_body @getter _zstd.ZstdDict.as_prefix @@ -211,15 +208,16 @@ Load as a prefix to compressor/decompressor. Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix) -1. Prefix is compatible with long distance matching, while dictionary is not. -2. It only works for the first frame, then the compressor/decompressor will - return to no prefix state. +1. Prefix is compatible with long distance matching, while + dictionary is not. +2. It only works for the first frame, then the + compressor/decompressor will return to no prefix state. 3. When decompressing, must use the same prefix as when compressing. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) -/*[clinic end generated code: output=6f7130c356595a16 input=77966c012d15e6ab]*/ +/*[clinic end generated code: output=6f7130c356595a16 input=192681a899c6fad0]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX); } diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 56a2a98fe42731a..9f613927be63159 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -8,7 +8,7 @@ #endif #include "Python.h" -#include "pycore_bytesobject.h" // _PyBytes_Repeat +#include "pycore_bytesobject.h" // _PyBytes_RepeatBuffer #include "pycore_call.h" // _PyObject_CallMethod() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_floatobject.h" // _PY_FLOAT_BIG_ENDIAN @@ -33,7 +33,7 @@ static struct PyModuleDef arraymodule; * functions aren't visible yet. */ struct arraydescr { - const char *typecode; + char typecode[3]; // big enough to store "Zd\0" int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); @@ -68,6 +68,7 @@ typedef struct { PyObject *str_write; PyObject *str___dict__; PyObject *str_iter; + PyObject *typecodes; } array_state; static array_state * @@ -292,49 +293,6 @@ BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return 0; } -static PyObject * -u_getitem(arrayobject *ap, Py_ssize_t i) -{ - return PyUnicode_FromOrdinal(((wchar_t *) ap->ob_item)[i]); -} - -static int -u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) -{ - if (!PyUnicode_Check(v)) { - PyErr_Format(PyExc_TypeError, - "array item must be a unicode character, not %T", - v); - return -1; - } - - Py_ssize_t len = PyUnicode_AsWideChar(v, NULL, 0); - if (len != 2) { - if (PyUnicode_GET_LENGTH(v) != 1) { - PyErr_Format(PyExc_TypeError, - "array item must be a unicode character, " - "not a string of length %zd", - PyUnicode_GET_LENGTH(v)); - } - else { - PyErr_Format(PyExc_TypeError, - "string %A cannot be converted to " - "a single wchar_t character", - v); - } - return -1; - } - - wchar_t w; - len = PyUnicode_AsWideChar(v, &w, 1); - assert(len == 1); - - if (i >= 0) { - ((wchar_t *)ap->ob_item)[i] = w; - } - return 0; -} - static PyObject * w_getitem(arrayobject *ap, Py_ssize_t i) { @@ -750,7 +708,6 @@ cd_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) DEFINE_COMPAREITEMS(b, signed char) DEFINE_COMPAREITEMS(BB, unsigned char) -DEFINE_COMPAREITEMS(u, wchar_t) DEFINE_COMPAREITEMS(w, Py_UCS4) DEFINE_COMPAREITEMS(h, short) DEFINE_COMPAREITEMS(HH, unsigned short) @@ -769,7 +726,6 @@ DEFINE_COMPAREITEMS(QQ, unsigned long long) static const struct arraydescr descriptors[] = { {"b", 1, b_getitem, b_setitem, b_compareitems, 1, 1}, {"B", 1, BB_getitem, BB_setitem, BB_compareitems, 1, 0}, - {"u", sizeof(wchar_t), u_getitem, u_setitem, u_compareitems, 0, 0}, {"w", sizeof(Py_UCS4), w_getitem, w_setitem, w_compareitems, 0, 0,}, {"h", sizeof(short), h_getitem, h_setitem, h_compareitems, 1, 1}, {"H", sizeof(short), HH_getitem, HH_setitem, HH_compareitems, 1, 0}, @@ -782,11 +738,9 @@ static const struct arraydescr descriptors[] = { {"e", sizeof(short), e_getitem, e_setitem, NULL, 0, 0}, {"f", sizeof(float), f_getitem, f_setitem, NULL, 0, 0}, {"d", sizeof(double), d_getitem, d_setitem, NULL, 0, 0}, - {"F", 2*sizeof(float), cf_getitem, cf_setitem, NULL, 0, 0}, - {"D", 2*sizeof(double), cd_getitem, cd_setitem, NULL, 0, 0}, {"Zf", 2*sizeof(float), cf_getitem, cf_setitem, NULL, 0, 0}, {"Zd", 2*sizeof(double), cd_getitem, cd_setitem, NULL, 0, 0}, - {NULL, 0, 0, 0, 0, 0, 0} /* Sentinel */ + {"", 0, 0, 0, 0, 0, 0} /* Sentinel */ }; /**************************************************************************** @@ -1149,7 +1103,7 @@ array_repeat(PyObject *op, Py_ssize_t n) const Py_ssize_t oldbytes = array_length * a->ob_descr->itemsize; const Py_ssize_t newbytes = oldbytes * n; - _PyBytes_Repeat(np->ob_item, newbytes, a->ob_item, oldbytes); + _PyBytes_RepeatBuffer(np->ob_item, newbytes, a->ob_item, oldbytes); return (PyObject *)np; } @@ -1306,7 +1260,7 @@ array_inplace_repeat(PyObject *op, Py_ssize_t n) if (array_resize(self, n * array_size) == -1) return NULL; - _PyBytes_Repeat(self->ob_item, n*size, self->ob_item, size); + _PyBytes_RepeatBuffer(self->ob_item, n*size, self->ob_item, size); } return Py_NewRef(self); } @@ -1536,13 +1490,13 @@ array.array.buffer_info Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents. -The length should be multiplied by the itemsize attribute to calculate -the buffer length in bytes. +The length should be multiplied by the itemsize attribute to +calculate the buffer length in bytes. [clinic start generated code]*/ static PyObject * array_array_buffer_info_impl(arrayobject *self) -/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=63d9ad83ba60cda8]*/ +/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=c2771b9f6a8e1c86]*/ { PyObject* item1 = PyLong_FromVoidPtr(self->ob_item); if (item1 == NULL) { @@ -1574,19 +1528,18 @@ array_array_append_impl(arrayobject *self, PyObject *v) } /*[clinic input] -@permit_long_docstring_body array.array.byteswap Byteswap all items of the array. -If the items in the array are not 1, 2, 4, 8 or 16 bytes in size, RuntimeError -is raised. Note, that for complex types the order of +If the items in the array are not 1, 2, 4, 8 or 16 bytes in size, +RuntimeError is raised. Note, that for complex types the order of components (the real part, followed by imaginary part) is preserved. [clinic start generated code]*/ static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=aafda275f48191d0]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=8732f800e1b47bac]*/ { char *p; Py_ssize_t i; @@ -1612,9 +1565,7 @@ array_array_byteswap_impl(arrayobject *self) } break; case 8: - if (strcmp(self->ob_descr->typecode, "F") != 0 - && strcmp(self->ob_descr->typecode, "Zf") != 0) - { + if (strcmp(self->ob_descr->typecode, "Zf") != 0) { for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 8) { char p0 = p[0]; char p1 = p[1]; @@ -1648,8 +1599,7 @@ array_array_byteswap_impl(arrayobject *self) } break; case 16: - assert(strcmp(self->ob_descr->typecode, "D") == 0 - || strcmp(self->ob_descr->typecode, "Zd") == 0); + assert(strcmp(self->ob_descr->typecode, "Zd") == 0); for (p = self->ob_item, i = Py_SIZE(self); --i >= 0; p += 8) { char t0 = p[0]; char t1 = p[1]; @@ -1972,7 +1922,6 @@ array_array_tobytes_impl(arrayobject *self) } /*[clinic input] -@permit_long_docstring_body array.array.fromunicode ustr: unicode @@ -1980,89 +1929,67 @@ array.array.fromunicode Extends this array with data from the unicode string ustr. -The array must be a unicode type array; otherwise a ValueError is raised. -Use array.frombytes(ustr.encode(...)) to append Unicode data to an array of -some other type. +The array must be a unicode type array; otherwise a ValueError is +raised. Use array.frombytes(ustr.encode(...)) to append Unicode +data to an array of some other type. [clinic start generated code]*/ static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) -/*[clinic end generated code: output=24359f5e001a7f2b input=158d47c302f27ca1]*/ +/*[clinic end generated code: output=24359f5e001a7f2b input=01fa592ec7b948b6]*/ { const char *typecode = self->ob_descr->typecode; - if (strcmp(typecode, "u") != 0 && strcmp(typecode, "w") != 0) { + if (strcmp(typecode, "w") != 0) { PyErr_SetString(PyExc_ValueError, "fromunicode() may only be called on " - "unicode type arrays ('u' or 'w')"); + "unicode type ('w') arrays"); return NULL; } - if (strcmp(typecode, "u") == 0) { - Py_ssize_t ustr_length = PyUnicode_AsWideChar(ustr, NULL, 0); - assert(ustr_length > 0); - if (ustr_length > 1) { - ustr_length--; /* trim trailing NUL character */ - Py_ssize_t old_size = Py_SIZE(self); - if (array_resize(self, old_size + ustr_length) == -1) { - return NULL; - } + Py_ssize_t ustr_length = PyUnicode_GetLength(ustr); + Py_ssize_t old_size = Py_SIZE(self); + Py_ssize_t new_size = old_size + ustr_length; - // must not fail - PyUnicode_AsWideChar( - ustr, ((wchar_t *)self->ob_item) + old_size, ustr_length); - } + if (new_size < 0 || (size_t)new_size > PY_SSIZE_T_MAX / sizeof(Py_UCS4)) { + return PyErr_NoMemory(); } - else { // typecode == "w" - Py_ssize_t ustr_length = PyUnicode_GetLength(ustr); - Py_ssize_t old_size = Py_SIZE(self); - Py_ssize_t new_size = old_size + ustr_length; - - if (new_size < 0 || (size_t)new_size > PY_SSIZE_T_MAX / sizeof(Py_UCS4)) { - return PyErr_NoMemory(); - } - if (array_resize(self, new_size) == -1) { - return NULL; - } - - // must not fail - Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)self->ob_item) + old_size, - ustr_length, 0); - assert(u != NULL); - (void)u; // Suppress unused_variable warning. + if (array_resize(self, new_size) == -1) { + return NULL; } + // must not fail + Py_UCS4 *u = PyUnicode_AsUCS4(ustr, ((Py_UCS4*)self->ob_item) + old_size, + ustr_length, 0); + assert(u != NULL); + (void)u; // Suppress unused_variable warning. + Py_RETURN_NONE; } /*[clinic input] -@permit_long_docstring_body array.array.tounicode Extends this array with data from the unicode string ustr. -Convert the array to a unicode string. The array must be a unicode type array; -otherwise a ValueError is raised. Use array.tobytes().decode() to obtain a -unicode string from an array of some other type. +Convert the array to a unicode string. The array must be a unicode +type array; otherwise a ValueError is raised. Use +array.tobytes().decode() to obtain a unicode string from an array of +some other type. [clinic start generated code]*/ static PyObject * array_array_tounicode_impl(arrayobject *self) -/*[clinic end generated code: output=08e442378336e1ef input=6690997213d219db]*/ +/*[clinic end generated code: output=08e442378336e1ef input=d4d5f398aa71a2be]*/ { const char *typecode = self->ob_descr->typecode; - if (strcmp(typecode, "u") != 0 && strcmp(typecode, "w") != 0) { + if (strcmp(typecode, "w") != 0) { PyErr_SetString(PyExc_ValueError, - "tounicode() may only be called on unicode type arrays ('u' or 'w')"); + "tounicode() may only be called on unicode type ('w') arrays"); return NULL; } - if (strcmp(typecode, "u") == 0) { - return PyUnicode_FromWideChar((wchar_t *) self->ob_item, Py_SIZE(self)); - } - else { // typecode == "w" - int byteorder = 0; // native byteorder - return PyUnicode_DecodeUTF32((const char *) self->ob_item, Py_SIZE(self) * 4, - NULL, &byteorder); - } + int byteorder = 0; // native byteorder + return PyUnicode_DecodeUTF32((const char *) self->ob_item, Py_SIZE(self) * 4, + NULL, &byteorder); } /*[clinic input] @@ -2138,15 +2065,6 @@ typecode_to_mformat_code(const char *typecode) case 'B': return UNSIGNED_INT8; - case 'u': - if (sizeof(wchar_t) == 2) { - return UTF16_LE + is_big_endian; - } - if (sizeof(wchar_t) == 4) { - return UTF32_LE + is_big_endian; - } - return UNKNOWN_FORMAT; - case 'w': return UTF32_LE + is_big_endian; @@ -2159,14 +2077,6 @@ typecode_to_mformat_code(const char *typecode) case 'd': return _PY_FLOAT_BIG_ENDIAN ? IEEE_754_DOUBLE_BE : IEEE_754_DOUBLE_LE; - case 'F': - return _PY_FLOAT_BIG_ENDIAN ? \ - IEEE_754_FLOAT_COMPLEX_BE : IEEE_754_FLOAT_COMPLEX_LE; - - case 'D': - return _PY_FLOAT_BIG_ENDIAN ? \ - IEEE_754_DOUBLE_COMPLEX_BE : IEEE_754_DOUBLE_COMPLEX_LE; - case 'Z': { switch (typecode[1]) { case 'f': @@ -2311,12 +2221,12 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, arraytype->tp_name, state->ArrayType->tp_name); return NULL; } - for (descr = descriptors; descr->typecode != NULL; descr++) { + for (descr = descriptors; descr->typecode[0] != 0; descr++) { if (strcmp(descr->typecode, typecode) == 0) { break; } } - if (descr->typecode == NULL) { + if (descr->typecode[0] == 0) { PyErr_SetString(PyExc_ValueError, "second argument must be a valid type code"); return NULL; @@ -2513,7 +2423,7 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, * * XXX: Is it possible to write a unit test for this? */ - for (descr = descriptors; descr->typecode != NULL; descr++) { + for (descr = descriptors; descr->typecode[0] != 0; descr++) { if (descr->is_integer_type && (size_t)descr->itemsize == mf_descr.size && descr->is_signed == mf_descr.is_signed) @@ -2691,7 +2601,8 @@ static PyMethodDef array_methods[] = { ARRAY_ARRAY_TOBYTES_METHODDEF ARRAY_ARRAY_TOUNICODE_METHODDEF ARRAY_ARRAY___SIZEOF___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Arrays are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2709,7 +2620,7 @@ array_repr(PyObject *op) return PyUnicode_FromFormat("%s('%s')", _PyType_Name(Py_TYPE(a)), typecode); } - if (strcmp(typecode, "u") == 0 || strcmp(typecode, "w") == 0) { + if (strcmp(typecode, "w") == 0) { v = array_array_tounicode_impl(a); } else { v = array_array_tolist_impl(a); @@ -2979,9 +2890,6 @@ array_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) view->internal = NULL; if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { view->format = (char *)self->ob_descr->typecode; - if (sizeof(wchar_t) >= 4 && strcmp(self->ob_descr->typecode, "u") == 0) { - view->format = "w"; - } } self->ob_exports++; @@ -3016,16 +2924,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - if (strcmp(s, "u") == 0) { - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "The 'u' type code is deprecated and " - "will be removed in Python 3.16", - 1)) { - return NULL; - } - } - - bool is_unicode = (strcmp(s, "u") == 0 || strcmp(s, "w") == 0); + bool is_unicode = (strcmp(s, "w") == 0); if (initial && !is_unicode) { if (PyUnicode_Check(initial)) { @@ -3035,7 +2934,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } else if (array_Check(initial, state)) { const char *is = ((arrayobject*)initial)->ob_descr->typecode; - if (strcmp(is, "u") == 0 || strcmp(is, "w") == 0) { + if (strcmp(is, "w") == 0) { PyErr_Format(PyExc_TypeError, "cannot use a unicode array to " "initialize an array with typecode '%s'", s); return NULL; @@ -3060,7 +2959,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) */ initial = NULL; } - for (descr = descriptors; descr->typecode != NULL; descr++) { + for (descr = descriptors; descr->typecode[0] != 0; descr++) { if (strcmp(descr->typecode, s) == 0) { PyObject *a; Py_ssize_t len; @@ -3111,43 +3010,21 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(v); } else if (initial != NULL && PyUnicode_Check(initial)) { - if (strcmp(s, "u") == 0) { - Py_ssize_t n; - wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); - if (ustr == NULL) { - Py_DECREF(a); - Py_XDECREF(it); - return NULL; - } - - if (n > 0) { - arrayobject *self = (arrayobject *)a; - // self->ob_item may be NULL but it is safe. - PyMem_Free(self->ob_item); - self->ob_item = (char *)ustr; - Py_SET_SIZE(self, n); - self->allocated = n; - } - else { - PyMem_Free(ustr); - } + assert(strcmp(descr->typecode, "w") == 0); + Py_ssize_t n = PyUnicode_GET_LENGTH(initial); + Py_UCS4 *ustr = PyUnicode_AsUCS4Copy(initial); + if (ustr == NULL) { + Py_DECREF(a); + Py_XDECREF(it); + return NULL; } - else { // s == "w" - Py_ssize_t n = PyUnicode_GET_LENGTH(initial); - Py_UCS4 *ustr = PyUnicode_AsUCS4Copy(initial); - if (ustr == NULL) { - Py_DECREF(a); - Py_XDECREF(it); - return NULL; - } - arrayobject *self = (arrayobject *)a; - // self->ob_item may be NULL but it is safe. - PyMem_Free(self->ob_item); - self->ob_item = (char *)ustr; - Py_SET_SIZE(self, n); - self->allocated = n; - } + arrayobject *self = (arrayobject *)a; + // self->ob_item may be NULL but it is safe. + PyMem_Free(self->ob_item); + self->ob_item = (char *)ustr; + Py_SET_SIZE(self, n); + self->allocated = n; } else if (initial != NULL && array_Check(initial, state) && len > 0) { arrayobject *self = (arrayobject *)a; @@ -3166,8 +3043,18 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } } Py_XDECREF(it); - PyErr_SetString(PyExc_ValueError, - "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, e, f, d, F, D, Zf or Zd)"); + + PyObject *sep = PyUnicode_FromString(", "); + if (sep == NULL) { + return NULL; + } + PyObject *msg = PyObject_CallMethod(sep, "join", "(O)", state->typecodes); + Py_DECREF(sep); + if (msg == NULL) { + return NULL; + } + PyErr_Format(PyExc_ValueError, "bad typecode (must be: %S)", msg); + Py_DECREF(msg); return NULL; } @@ -3193,7 +3080,6 @@ The following type codes are defined:\n\ Type code C Type Minimum size in bytes\n\ 'b' signed integer 1\n\ 'B' unsigned integer 1\n\ - 'u' Unicode character 2 (see note)\n\ 'h' signed integer 2\n\ 'H' unsigned integer 2\n\ 'i' signed integer 2\n\ @@ -3205,14 +3091,9 @@ The following type codes are defined:\n\ 'e' 16-bit IEEE floats 2\n\ 'f' floating-point 4\n\ 'd' floating-point 8\n\ - 'F' float complex 8\n\ - 'D' double complex 16\n\ 'Zf' float complex 8\n\ 'Zd' double complex 16\n\ \n\ -NOTE: The 'u' typecode corresponds to Python's unicode character. On\n\ -narrow builds this is 2-bytes on wide builds this is 4-bytes.\n\ -\n\ NOTE: The 'q' and 'Q' type codes are only available if the platform\n\ C compiler used to build Python supports 'long long', or, on Windows,\n\ '__int64'.\n\ @@ -3454,6 +3335,7 @@ array_traverse(PyObject *module, visitproc visit, void *arg) Py_VISIT(state->ArrayType); Py_VISIT(state->ArrayIterType); Py_VISIT(state->array_reconstructor); + Py_VISIT(state->typecodes); return 0; } @@ -3468,6 +3350,7 @@ array_clear(PyObject *module) Py_CLEAR(state->str_write); Py_CLEAR(state->str___dict__); Py_CLEAR(state->str_iter); + Py_CLEAR(state->typecodes); return 0; } @@ -3546,7 +3429,7 @@ array_modexec(PyObject *m) if (typecodes == NULL) { return -1; } - for (descr = descriptors; descr->typecode != NULL; descr++) { + for (descr = descriptors; descr->typecode[0] != 0; descr++) { PyObject *typecode = PyUnicode_DecodeASCII(descr->typecode, strlen(descr->typecode), NULL); if (typecode == NULL) { Py_DECREF(typecodes); @@ -3564,6 +3447,7 @@ array_modexec(PyObject *m) if (tuple == NULL) { return -1; } + state->typecodes = Py_NewRef(tuple); if (PyModule_Add(m, "typecodes", tuple) < 0) { return -1; } diff --git a/Modules/binascii.c b/Modules/binascii.c index 673dca6ee134bd8..0e7af135a6f6ce4 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1057,7 +1057,8 @@ binascii.a2b_ascii85 foldspaces: bool = False Allow 'y' as a short form encoding four spaces. adobe: bool = False - Expect data to be wrapped in '<~' and '~>' as in Adobe Ascii85. + Expect data to be terminated with '~>' as in Adobe Ascii85, and + optionally accept leading '<~'. ignorechars: Py_buffer = b'' A byte string containing characters to ignore from the input. canonical: bool = False @@ -1069,7 +1070,7 @@ Decode Ascii85 data. static PyObject * binascii_a2b_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, int adobe, Py_buffer *ignorechars, int canonical) -/*[clinic end generated code: output=09b35f1eac531357 input=dd050604ed30199e]*/ +/*[clinic end generated code: output=09b35f1eac531357 input=08eab2e53c62f1a8]*/ { const unsigned char *ascii_data = data->buf; Py_ssize_t ascii_len = data->len; @@ -1264,7 +1265,7 @@ binascii.b2a_ascii85 wrapcol: size_t = 0 Split result into lines of provided width. pad: bool = False - Pad input to a multiple of 4 before encoding. + Retain zero-padding bytes at end of output. adobe: bool = False Wrap result in '<~' and '~>' as in Adobe Ascii85. @@ -1274,7 +1275,7 @@ Ascii85-encode data. static PyObject * binascii_b2a_ascii85_impl(PyObject *module, Py_buffer *data, int foldspaces, size_t wrapcol, int pad, int adobe) -/*[clinic end generated code: output=5ce8fdee843073f4 input=791da754508c7d17]*/ +/*[clinic end generated code: output=5ce8fdee843073f4 input=a77e31d63517bf19]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; @@ -1539,7 +1540,7 @@ binascii.b2a_base85 / * pad: bool = False - Pad input to a multiple of 4 before encoding. + Retain zero-padding bytes at end of output. wrapcol: size_t = 0 alphabet: Py_buffer(c_default="{NULL, NULL}") = BASE85_ALPHABET @@ -1549,7 +1550,7 @@ Base85-code line of data. static PyObject * binascii_b2a_base85_impl(PyObject *module, Py_buffer *data, int pad, size_t wrapcol, Py_buffer *alphabet) -/*[clinic end generated code: output=98b962ed52c776a4 input=1b20b0bd6572691b]*/ +/*[clinic end generated code: output=98b962ed52c776a4 input=54886d05128d41a8]*/ { const unsigned char *bin_data = data->buf; Py_ssize_t bin_len = data->len; diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b3663180d726e5e..32588b0561e1acc 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -14,10 +14,11 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "\n" "Return an encoded string version of \'input\'.\n" "\n" -"\'errors\' may be given to set a different error handling scheme. Default is\n" -"\'strict\' meaning that encoding errors raise a UnicodeEncodeError. Other possible\n" -"values are \'ignore\', \'replace\' and \'xmlcharrefreplace\' as well as any other name\n" -"registered with codecs.register_error that can handle UnicodeEncodeErrors."); +"\'errors\' may be given to set a different error handling scheme.\n" +"Default is \'strict\' meaning that encoding errors raise\n" +"a UnicodeEncodeError. Other possible values are \'ignore\', \'replace\'\n" +"and \'xmlcharrefreplace\' as well as any other name registered with\n" +"codecs.register_error that can handle UnicodeEncodeErrors."); #define _MULTIBYTECODEC_MULTIBYTECODEC_ENCODE_METHODDEF \ {"encode", _PyCFunction_CAST(_multibytecodec_MultibyteCodec_encode), METH_FASTCALL|METH_KEYWORDS, _multibytecodec_MultibyteCodec_encode__doc__}, @@ -103,9 +104,10 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_decode__doc__, "\n" "Decodes \'input\'.\n" "\n" -"\'errors\' may be given to set a different error handling scheme. Default is\n" -"\'strict\' meaning that encoding errors raise a UnicodeDecodeError. Other possible\n" -"values are \'ignore\' and \'replace\' as well as any other name registered with\n" +"\'errors\' may be given to set a different error handling scheme.\n" +"Default is \'strict\' meaning that encoding errors raise\n" +"a UnicodeDecodeError. Other possible values are \'ignore\' and\n" +"\'replace\' as well as any other name registered with\n" "codecs.register_error that is able to handle UnicodeDecodeErrors.\""); #define _MULTIBYTECODEC_MULTIBYTECODEC_DECODE_METHODDEF \ @@ -696,4 +698,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=014f4f6bb9d29594 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a84b1544d7d01abb input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index f1124147e2b0a78..32c96c9d2cb3cde 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -582,7 +582,6 @@ multibytecodec_encode(const MultibyteCodec *codec, } /*[clinic input] -@permit_long_docstring_body _multibytecodec.MultibyteCodec.encode input: object @@ -590,17 +589,18 @@ _multibytecodec.MultibyteCodec.encode Return an encoded string version of 'input'. -'errors' may be given to set a different error handling scheme. Default is -'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible -values are 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name -registered with codecs.register_error that can handle UnicodeEncodeErrors. +'errors' may be given to set a different error handling scheme. +Default is 'strict' meaning that encoding errors raise +a UnicodeEncodeError. Other possible values are 'ignore', 'replace' +and 'xmlcharrefreplace' as well as any other name registered with +codecs.register_error that can handle UnicodeEncodeErrors. [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=0980aede2c564df8]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=980002ed1447697b]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -648,7 +648,6 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, } /*[clinic input] -@permit_long_docstring_body _multibytecodec.MultibyteCodec.decode input: Py_buffer @@ -656,9 +655,10 @@ _multibytecodec.MultibyteCodec.decode Decodes 'input'. -'errors' may be given to set a different error handling scheme. Default is -'strict' meaning that encoding errors raise a UnicodeDecodeError. Other possible -values are 'ignore' and 'replace' as well as any other name registered with +'errors' may be given to set a different error handling scheme. +Default is 'strict' meaning that encoding errors raise +a UnicodeDecodeError. Other possible values are 'ignore' and +'replace' as well as any other name registered with codecs.register_error that is able to handle UnicodeDecodeErrors." [clinic start generated code]*/ @@ -666,7 +666,7 @@ static PyObject * _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, Py_buffer *input, const char *errors) -/*[clinic end generated code: output=ff419f65bad6cc77 input=2c657ef914600c7c]*/ +/*[clinic end generated code: output=ff419f65bad6cc77 input=dbf93d8bb98ca440]*/ { MultibyteCodec_State state; MultibyteDecodeBuffer buf; diff --git a/Modules/clinic/_abc.c.h b/Modules/clinic/_abc.c.h index 04681fa2206a2a5..fa1c57dc26bf853 100644 --- a/Modules/clinic/_abc.c.h +++ b/Modules/clinic/_abc.c.h @@ -146,9 +146,9 @@ PyDoc_STRVAR(_abc_get_cache_token__doc__, "\n" "Returns the current ABC cache token.\n" "\n" -"The token is an opaque object (supporting equality testing) identifying the\n" -"current version of the ABC cache for virtual subclasses. The token changes\n" -"with every call to register() on any ABC."); +"The token is an opaque object (supporting equality testing) identifying\n" +"the current version of the ABC cache for virtual subclasses. The token\n" +"changes with every call to register() on any ABC."); #define _ABC_GET_CACHE_TOKEN_METHODDEF \ {"get_cache_token", (PyCFunction)_abc_get_cache_token, METH_NOARGS, _abc_get_cache_token__doc__}, @@ -161,4 +161,4 @@ _abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _abc_get_cache_token_impl(module); } -/*[clinic end generated code: output=1989b6716c950e17 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b05d599656aeb1e1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 66953d74213b66d..f07a09df5ac7ae0 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -90,7 +90,8 @@ PyDoc_STRVAR(_asyncio_Future_result__doc__, "\n" "If the future has been cancelled, raises CancelledError. If the\n" "future\'s result isn\'t yet available, raises InvalidStateError. If\n" -"the future is done and has an exception set, this exception is raised."); +"the future is done and has an exception set, this exception is\n" +"raised."); #define _ASYNCIO_FUTURE_RESULT_METHODDEF \ {"result", (PyCFunction)_asyncio_Future_result, METH_NOARGS, _asyncio_Future_result__doc__}, @@ -250,8 +251,8 @@ PyDoc_STRVAR(_asyncio_Future_add_done_callback__doc__, "\n" "Add a callback to be run when the future becomes done.\n" "\n" -"The callback is called with a single argument - the future object. If\n" -"the future is already done when this is called, the callback is\n" +"The callback is called with a single argument - the future object.\n" +"If the future is already done when this is called, the callback is\n" "scheduled with call_soon."); #define _ASYNCIO_FUTURE_ADD_DONE_CALLBACK_METHODDEF \ @@ -371,9 +372,9 @@ PyDoc_STRVAR(_asyncio_Future_cancel__doc__, "\n" "Cancel the future and schedule callbacks.\n" "\n" -"If the future is already done or cancelled, return False. Otherwise,\n" -"change the future\'s state to cancelled, schedule the callbacks and\n" -"return True."); +"If the future is already done or cancelled, return False.\n" +"Otherwise, change the future\'s state to cancelled, schedule the\n" +"callbacks and return True."); #define _ASYNCIO_FUTURE_CANCEL_METHODDEF \ {"cancel", _PyCFunction_CAST(_asyncio_Future_cancel), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_cancel__doc__}, @@ -465,8 +466,8 @@ PyDoc_STRVAR(_asyncio_Future_done__doc__, "\n" "Return True if the future is done.\n" "\n" -"Done means either that a result / exception are available, or that the\n" -"future was cancelled."); +"Done means either that a result / exception are available, or that\n" +"the future was cancelled."); #define _ASYNCIO_FUTURE_DONE_METHODDEF \ {"done", (PyCFunction)_asyncio_Future_done, METH_NOARGS, _asyncio_Future_done__doc__}, @@ -2232,4 +2233,4 @@ _asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=b69948ed810591d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=32996fb47c48245b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 8f3492cd54b5f2f..454131faf4112b7 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -16,8 +16,8 @@ PyDoc_STRVAR(_bisect_bisect_right__doc__, "Return the index where to insert item x in list a, assuming a is sorted.\n" "\n" "The return value i is such that all e in a[:i] have e <= x, and all e in\n" -"a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will\n" -"insert just after the rightmost x already there.\n" +"a[i:] have e > x. So if x already appears in the list, a.insert(i, x)\n" +"will insert just after the rightmost x already there.\n" "\n" "Optional args lo (default 0) and hi (default len(a)) bound the\n" "slice of a to be searched.\n" @@ -245,8 +245,8 @@ PyDoc_STRVAR(_bisect_bisect_left__doc__, "Return the index where to insert item x in list a, assuming a is sorted.\n" "\n" "The return value i is such that all e in a[:i] have e < x, and all e in\n" -"a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will\n" -"insert just before the leftmost x already there.\n" +"a[i:] have e >= x. So if x already appears in the list, a.insert(i, x)\n" +"will insert just before the leftmost x already there.\n" "\n" "Optional args lo (default 0) and hi (default len(a)) bound the\n" "slice of a to be searched.\n" @@ -466,4 +466,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=a3c44ed440dd6d81 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=62345f14c5c01639 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index 2bc6524b6a973b8..30f2c7965e73ae1 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -116,18 +116,19 @@ PyDoc_STRVAR(_bz2_BZ2Decompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define _BZ2_BZ2DECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_bz2_BZ2Decompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _bz2_BZ2Decompressor_decompress__doc__}, @@ -237,4 +238,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=552ac6d4c5a101b7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1dce5396d592bad7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index 9e2a7950ebde64d..4a40dc660b621cc 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -14,9 +14,10 @@ PyDoc_STRVAR(_codecs_register__doc__, "\n" "Register a codec search function.\n" "\n" -"Search functions are expected to take one argument, the encoding name in\n" -"all lower case letters, and either return None, or a tuple of functions\n" -"(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object)."); +"Search functions are expected to take one argument, the encoding\n" +"name in all lower case letters, and either return None, or a tuple\n" +"of functions (encoder, decoder, stream_reader, stream_writer) (or\n" +"a CodecInfo object)."); #define _CODECS_REGISTER_METHODDEF \ {"register", (PyCFunction)_codecs_register, METH_O, _codecs_register__doc__}, @@ -76,10 +77,10 @@ PyDoc_STRVAR(_codecs_encode__doc__, "Encodes obj using the codec registered for encoding.\n" "\n" "The default encoding is \'utf-8\'. errors may be given to set a\n" -"different error handling scheme. Default is \'strict\' meaning that encoding\n" -"errors raise a ValueError. Other possible values are \'ignore\', \'replace\'\n" -"and \'backslashreplace\' as well as any other name registered with\n" -"codecs.register_error that can handle ValueErrors."); +"different error handling scheme. Default is \'strict\' meaning that\n" +"encoding errors raise a ValueError. Other possible values are \'ignore\',\n" +"\'replace\' and \'backslashreplace\' as well as any other name registered\n" +"with codecs.register_error that can handle ValueErrors."); #define _CODECS_ENCODE_METHODDEF \ {"encode", _PyCFunction_CAST(_codecs_encode), METH_FASTCALL|METH_KEYWORDS, _codecs_encode__doc__}, @@ -179,10 +180,10 @@ PyDoc_STRVAR(_codecs_decode__doc__, "Decodes obj using the codec registered for encoding.\n" "\n" "Default encoding is \'utf-8\'. errors may be given to set a\n" -"different error handling scheme. Default is \'strict\' meaning that encoding\n" -"errors raise a ValueError. Other possible values are \'ignore\', \'replace\'\n" -"and \'backslashreplace\' as well as any other name registered with\n" -"codecs.register_error that can handle ValueErrors."); +"different error handling scheme. Default is \'strict\' meaning that\n" +"encoding errors raise a ValueError. Other possible values are \'ignore\',\n" +"\'replace\' and \'backslashreplace\' as well as any other name registered\n" +"with codecs.register_error that can handle ValueErrors."); #define _CODECS_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(_codecs_decode), METH_FASTCALL|METH_KEYWORDS, _codecs_decode__doc__}, @@ -2649,8 +2650,9 @@ PyDoc_STRVAR(_codecs_register_error__doc__, "Register the specified error handler under the name errors.\n" "\n" "handler must be a callable object, that will be called with an exception\n" -"instance containing information about the location of the encoding/decoding\n" -"error and must return a (replacement, new position) tuple."); +"instance containing information about the location of the\n" +"encoding/decoding error and must return a (replacement, new position)\n" +"tuple."); #define _CODECS_REGISTER_ERROR_METHODDEF \ {"register_error", _PyCFunction_CAST(_codecs_register_error), METH_FASTCALL, _codecs_register_error__doc__}, @@ -2745,8 +2747,8 @@ PyDoc_STRVAR(_codecs_lookup_error__doc__, "\n" "lookup_error(errors) -> handler\n" "\n" -"Return the error handler for the specified error handling name or raise a\n" -"LookupError, if no handler exists under this name."); +"Return the error handler for the specified error handling name or raise\n" +"a LookupError, if no handler exists under this name."); #define _CODECS_LOOKUP_ERROR_METHODDEF \ {"lookup_error", (PyCFunction)_codecs_lookup_error, METH_O, _codecs_lookup_error__doc__}, @@ -2866,4 +2868,4 @@ _codecs__normalize_encoding(PyObject *module, PyObject *const *args, Py_ssize_t #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=a968c493bb28be3e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=505edef891a06329 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index b5c315c680e7821..6c60678a6fbd51a 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -340,7 +340,7 @@ deque_index(PyObject *deque, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *v; Py_ssize_t start = 0; - Py_ssize_t stop = Py_SIZE(deque); + Py_ssize_t stop = PY_SSIZE_T_MAX; if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { goto exit; @@ -632,4 +632,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b9d4d647c221cb9f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f5a388add99d3d15 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 75cf067c8aa822c..316264db80a6582 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -28,7 +28,8 @@ PyDoc_STRVAR(_curses_panel_panel_hide__doc__, "\n" "Hide the panel.\n" "\n" -"This does not delete the object, it just makes the window on screen invisible."); +"This does not delete the object, it just makes the window on screen\n" +"invisible."); #define _CURSES_PANEL_PANEL_HIDE_METHODDEF \ {"hide", (PyCFunction)_curses_panel_panel_hide, METH_NOARGS, _curses_panel_panel_hide__doc__}, @@ -46,7 +47,9 @@ PyDoc_STRVAR(_curses_panel_panel_show__doc__, "show($self, /)\n" "--\n" "\n" -"Display the panel (which might have been hidden)."); +"Display the panel (which might have been hidden).\n" +"\n" +"The panel is placed on top of the panel stack."); #define _CURSES_PANEL_PANEL_SHOW_METHODDEF \ {"show", (PyCFunction)_curses_panel_panel_show, METH_NOARGS, _curses_panel_panel_show__doc__}, @@ -279,7 +282,9 @@ PyDoc_STRVAR(_curses_panel_new_panel__doc__, "new_panel($module, win, /)\n" "--\n" "\n" -"Return a panel object, associating it with the given window win."); +"Return a panel object, associating it with the given window win.\n" +"\n" +"The new panel is placed on top of the panel stack."); #define _CURSES_PANEL_NEW_PANEL_METHODDEF \ {"new_panel", (PyCFunction)_curses_panel_new_panel, METH_O, _curses_panel_new_panel__doc__}, @@ -328,7 +333,8 @@ PyDoc_STRVAR(_curses_panel_update_panels__doc__, "\n" "Updates the virtual screen after changes in the panel stack.\n" "\n" -"This does not call curses.doupdate(), so you\'ll have to do this yourself."); +"This does not call curses.doupdate(), so you\'ll have to do this\n" +"yourself."); #define _CURSES_PANEL_UPDATE_PANELS_METHODDEF \ {"update_panels", (PyCFunction)_curses_panel_update_panels, METH_NOARGS, _curses_panel_update_panels__doc__}, @@ -341,4 +347,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=db2fe491582784aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ec3fe619150e512e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index e6f9798cdf12498..49f30a35656b485 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -422,10 +422,10 @@ PyDoc_STRVAR(_curses_window_border__doc__, " br\n" " Bottom-right corner.\n" "\n" -"Each parameter specifies the character to use for a specific part of the\n" -"border. The characters can be specified as integers or as one-character\n" -"strings. A 0 value for any parameter will cause the default character to be\n" -"used for that parameter."); +"Each parameter specifies the character to use for a specific part of\n" +"the border. The characters can be specified as integers or as\n" +"one-character strings. A 0 value for any parameter will cause the\n" +"default character to be used for that parameter."); #define _CURSES_WINDOW_BORDER_METHODDEF \ {"border", _PyCFunction_CAST(_curses_window_border), METH_FASTCALL, _curses_window_border__doc__}, @@ -500,8 +500,9 @@ PyDoc_STRVAR(_curses_window_box__doc__, " horch\n" " Top and bottom side.\n" "\n" -"Similar to border(), but both ls and rs are verch and both ts and bs are\n" -"horch. The default corner characters are always used by this function."); +"Similar to border(), but both ls and rs are verch and both ts and bs\n" +"are horch. The default corner characters are always used by this\n" +"function."); #define _CURSES_WINDOW_BOX_METHODDEF \ {"box", (PyCFunction)_curses_window_box, METH_VARARGS, _curses_window_box__doc__}, @@ -539,12 +540,15 @@ _curses_window_box(PyObject *self, PyObject *args) PyDoc_STRVAR(_curses_window_delch__doc__, "delch([y, x])\n" -"Delete any character at (y, x).\n" +"Delete the character under the cursor, or at (y, x) if specified.\n" "\n" " y\n" " Y-coordinate.\n" " x\n" -" X-coordinate."); +" X-coordinate.\n" +"\n" +"All characters to the right on the same line are shifted one\n" +"position left."); #define _CURSES_WINDOW_DELCH_METHODDEF \ {"delch", (PyCFunction)_curses_window_delch, METH_VARARGS, _curses_window_delch__doc__}, @@ -593,9 +597,9 @@ PyDoc_STRVAR(_curses_window_derwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"derwin() is the same as calling subwin(), except that begin_y and begin_x\n" -"are relative to the origin of the window, rather than relative to the entire\n" -"screen."); +"derwin() is the same as calling subwin(), except that begin_y and\n" +"begin_x are relative to the origin of the window, rather than\n" +"relative to the entire screen."); #define _CURSES_WINDOW_DERWIN_METHODDEF \ {"derwin", (PyCFunction)_curses_window_derwin, METH_VARARGS, _curses_window_derwin__doc__}, @@ -751,9 +755,10 @@ PyDoc_STRVAR(_curses_window_getch__doc__, " x\n" " X-coordinate.\n" "\n" -"The integer returned does not have to be in ASCII range: function keys,\n" -"keypad keys and so on return numbers higher than 256. In no-delay mode, -1\n" -"is returned if there is no input, else getch() waits until a key is pressed."); +"The integer returned does not have to be in ASCII range: function\n" +"keys, keypad keys and so on return numbers higher than 256. In\n" +"no-delay mode, -1 is returned if there is no input, else getch()\n" +"waits until a key is pressed."); #define _CURSES_WINDOW_GETCH_METHODDEF \ {"getch", (PyCFunction)_curses_window_getch, METH_VARARGS, _curses_window_getch__doc__}, @@ -798,9 +803,10 @@ PyDoc_STRVAR(_curses_window_getkey__doc__, " x\n" " X-coordinate.\n" "\n" -"Returning a string instead of an integer, as getch() does. Function keys,\n" -"keypad keys and other special keys return a multibyte string containing the\n" -"key name. In no-delay mode, an exception is raised if there is no input."); +"Returning a string instead of an integer, as getch() does. Function\n" +"keys, keypad keys and other special keys return a multibyte string\n" +"containing the key name. In no-delay mode, an exception is raised\n" +"if there is no input."); #define _CURSES_WINDOW_GETKEY_METHODDEF \ {"getkey", (PyCFunction)_curses_window_getkey, METH_VARARGS, _curses_window_getkey__doc__}, @@ -969,8 +975,8 @@ PyDoc_STRVAR(_curses_window_insch__doc__, " attr\n" " Attributes for the character.\n" "\n" -"All characters to the right of the cursor are shifted one position right, with\n" -"the rightmost characters on the line being lost."); +"All characters to the right of the cursor are shifted one position\n" +"right, with the rightmost characters on the line being lost."); #define _CURSES_WINDOW_INSCH_METHODDEF \ {"insch", (PyCFunction)_curses_window_insch, METH_VARARGS, _curses_window_insch__doc__}, @@ -1035,7 +1041,8 @@ PyDoc_STRVAR(_curses_window_inch__doc__, " x\n" " X-coordinate.\n" "\n" -"The bottom 8 bits are the character proper, and upper bits are the attributes."); +"The bottom 8 bits are the character proper, and upper bits are the\n" +"attributes."); #define _CURSES_WINDOW_INCH_METHODDEF \ {"inch", (PyCFunction)_curses_window_inch, METH_VARARGS, _curses_window_inch__doc__}, @@ -1084,11 +1091,11 @@ PyDoc_STRVAR(_curses_window_insstr__doc__, " attr\n" " Attributes for characters.\n" "\n" -"Insert a character string (as many characters as will fit on the line)\n" -"before the character under the cursor. All characters to the right of\n" -"the cursor are shifted right, with the rightmost characters on the line\n" -"being lost. The cursor position does not change (after moving to y, x,\n" -"if specified)."); +"Insert a character string (as many characters as will fit on the\n" +"line) before the character under the cursor. All characters to the\n" +"right of the cursor are shifted right, with the rightmost characters\n" +"on the line being lost. The cursor position does not change (after\n" +"moving to y, x, if specified)."); #define _CURSES_WINDOW_INSSTR_METHODDEF \ {"insstr", (PyCFunction)_curses_window_insstr, METH_VARARGS, _curses_window_insstr__doc__}, @@ -1159,12 +1166,12 @@ PyDoc_STRVAR(_curses_window_insnstr__doc__, " attr\n" " Attributes for characters.\n" "\n" -"Insert a character string (as many characters as will fit on the line)\n" -"before the character under the cursor, up to n characters. If n is zero\n" -"or negative, the entire string is inserted. All characters to the right\n" -"of the cursor are shifted right, with the rightmost characters on the line\n" -"being lost. The cursor position does not change (after moving to y, x, if\n" -"specified)."); +"Insert a character string (as many characters as will fit on the\n" +"line) before the character under the cursor, up to n characters. If\n" +"n is zero or negative, the entire string is inserted. All\n" +"characters to the right of the cursor are shifted right, with the\n" +"rightmost characters on the line being lost. The cursor position\n" +"does not change (after moving to y, x, if specified)."); #define _CURSES_WINDOW_INSNSTR_METHODDEF \ {"insnstr", (PyCFunction)_curses_window_insnstr, METH_VARARGS, _curses_window_insnstr__doc__}, @@ -1230,7 +1237,8 @@ PyDoc_STRVAR(_curses_window_is_linetouched__doc__, " line\n" " Line number.\n" "\n" -"Raise a curses.error exception if line is not valid for the given window."); +"Raise a curses.error exception if line is not valid for the given\n" +"window."); #define _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF \ {"is_linetouched", (PyCFunction)_curses_window_is_linetouched, METH_O, _curses_window_is_linetouched__doc__}, @@ -1260,9 +1268,9 @@ PyDoc_STRVAR(_curses_window_noutrefresh__doc__, "noutrefresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol])\n" "Mark for refresh but wait.\n" "\n" -"This function updates the data structure representing the desired state of the\n" -"window, but does not force an update of the physical screen. To accomplish\n" -"that, call doupdate()."); +"This function updates the data structure representing the desired\n" +"state of the window, but does not force an update of the physical\n" +"screen. To accomplish that, call doupdate()."); #define _CURSES_WINDOW_NOUTREFRESH_METHODDEF \ {"noutrefresh", (PyCFunction)_curses_window_noutrefresh, METH_VARARGS, _curses_window_noutrefresh__doc__}, @@ -1314,9 +1322,9 @@ PyDoc_STRVAR(_curses_window_noutrefresh__doc__, "\n" "Mark for refresh but wait.\n" "\n" -"This function updates the data structure representing the desired state of the\n" -"window, but does not force an update of the physical screen. To accomplish\n" -"that, call doupdate()."); +"This function updates the data structure representing the desired\n" +"state of the window, but does not force an update of the physical\n" +"screen. To accomplish that, call doupdate()."); #define _CURSES_WINDOW_NOUTREFRESH_METHODDEF \ {"noutrefresh", (PyCFunction)_curses_window_noutrefresh, METH_NOARGS, _curses_window_noutrefresh__doc__}, @@ -1336,14 +1344,15 @@ PyDoc_STRVAR(_curses_window_overlay__doc__, "overlay(destwin, [sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol])\n" "Overlay the window on top of destwin.\n" "\n" -"The windows need not be the same size, only the overlapping region is copied.\n" -"This copy is non-destructive, which means that the current background\n" -"character does not overwrite the old contents of destwin.\n" +"The windows need not be the same size, only the overlapping region\n" +"is copied. This copy is non-destructive, which means that the\n" +"current background character does not overwrite the old contents of\n" +"destwin.\n" "\n" -"To get fine-grained control over the copied region, the second form of\n" -"overlay() can be used. sminrow and smincol are the upper-left coordinates\n" -"of the source window, and the other variables mark a rectangle in the\n" -"destination window."); +"To get fine-grained control over the copied region, the second form\n" +"of overlay() can be used. sminrow and smincol are the upper-left\n" +"coordinates of the source window, and the other variables mark\n" +"a rectangle in the destination window."); #define _CURSES_WINDOW_OVERLAY_METHODDEF \ {"overlay", (PyCFunction)_curses_window_overlay, METH_VARARGS, _curses_window_overlay__doc__}, @@ -1394,14 +1403,15 @@ PyDoc_STRVAR(_curses_window_overwrite__doc__, " dmaxcol])\n" "Overwrite the window on top of destwin.\n" "\n" -"The windows need not be the same size, in which case only the overlapping\n" -"region is copied. This copy is destructive, which means that the current\n" -"background character overwrites the old contents of destwin.\n" +"The windows need not be the same size, in which case only the\n" +"overlapping region is copied. This copy is destructive, which means\n" +"that the current background character overwrites the old contents of\n" +"destwin.\n" "\n" -"To get fine-grained control over the copied region, the second form of\n" -"overwrite() can be used. sminrow and smincol are the upper-left coordinates\n" -"of the source window, the other variables mark a rectangle in the destination\n" -"window."); +"To get fine-grained control over the copied region, the second form\n" +"of overwrite() can be used. sminrow and smincol are the upper-left\n" +"coordinates of the source window, the other variables mark\n" +"a rectangle in the destination window."); #define _CURSES_WINDOW_OVERWRITE_METHODDEF \ {"overwrite", (PyCFunction)_curses_window_overwrite, METH_VARARGS, _curses_window_overwrite__doc__}, @@ -1520,16 +1530,17 @@ PyDoc_STRVAR(_curses_window_refresh__doc__, "Update the display immediately.\n" "\n" "Synchronize actual screen with previous drawing/deleting methods.\n" -"The 6 optional arguments can only be specified when the window is a pad\n" -"created with newpad(). The additional parameters are needed to indicate\n" -"what part of the pad and screen are involved. pminrow and pmincol specify\n" -"the upper left-hand corner of the rectangle to be displayed in the pad.\n" -"sminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to\n" -"be displayed on the screen. The lower right-hand corner of the rectangle to\n" -"be displayed in the pad is calculated from the screen coordinates, since the\n" -"rectangles must be the same size. Both rectangles must be entirely contained\n" -"within their respective structures. Negative values of pminrow, pmincol,\n" -"sminrow, or smincol are treated as if they were zero."); +"The 6 optional arguments can only be specified when the window is\n" +"a pad created with newpad(). The additional parameters are needed\n" +"to indicate what part of the pad and screen are involved. pminrow\n" +"and pmincol specify the upper left-hand corner of the rectangle to\n" +"be displayed in the pad. sminrow, smincol, smaxrow, and smaxcol\n" +"specify the edges of the rectangle to be displayed on the screen.\n" +"The lower right-hand corner of the rectangle to be displayed in the\n" +"pad is calculated from the screen coordinates, since the rectangles\n" +"must be the same size. Both rectangles must be entirely contained\n" +"within their respective structures. Negative values of pminrow,\n" +"pmincol, sminrow, or smincol are treated as if they were zero."); #define _CURSES_WINDOW_REFRESH_METHODDEF \ {"refresh", (PyCFunction)_curses_window_refresh, METH_VARARGS, _curses_window_refresh__doc__}, @@ -1627,8 +1638,8 @@ PyDoc_STRVAR(_curses_window_subwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"By default, the sub-window will extend from the specified position to the\n" -"lower right corner of the window."); +"By default, the sub-window will extend from the specified position\n" +"to the lower right corner of the window."); #define _CURSES_WINDOW_SUBWIN_METHODDEF \ {"subwin", (PyCFunction)_curses_window_subwin, METH_VARARGS, _curses_window_subwin__doc__}, @@ -1676,7 +1687,8 @@ PyDoc_STRVAR(_curses_window_scroll__doc__, " lines\n" " Number of lines to scroll.\n" "\n" -"Scroll upward if the argument is positive and downward if it is negative."); +"Scroll upward if the argument is positive and downward if it is\n" +"negative."); #define _CURSES_WINDOW_SCROLL_METHODDEF \ {"scroll", (PyCFunction)_curses_window_scroll, METH_VARARGS, _curses_window_scroll__doc__}, @@ -1715,8 +1727,9 @@ PyDoc_STRVAR(_curses_window_touchline__doc__, "touchline(start, count, [changed=True])\n" "Pretend count lines have been changed, starting with line start.\n" "\n" -"If changed is supplied, it specifies whether the affected lines are marked\n" -"as having been changed (changed=True) or unchanged (changed=False)."); +"If changed is supplied, it specifies whether the affected lines are\n" +"marked as having been changed (changed=True) or unchanged\n" +"(changed=False)."); #define _CURSES_WINDOW_TOUCHLINE_METHODDEF \ {"touchline", (PyCFunction)_curses_window_touchline, METH_VARARGS, _curses_window_touchline__doc__}, @@ -1831,7 +1844,13 @@ _curses_window_vline(PyObject *self, PyObject *args) PyDoc_STRVAR(_curses_filter__doc__, "filter($module, /)\n" "--\n" -"\n"); +"\n" +"Restrict screen updates to the current line.\n" +"\n" +"Must be called before initscr(). Afterwards curses confines the cursor\n" +"and screen updates to a single line, which is useful for enabling\n" +"character-at-a-time line editing without touching the rest of the\n" +"screen."); #define _CURSES_FILTER_METHODDEF \ {"filter", (PyCFunction)_curses_filter, METH_NOARGS, _curses_filter__doc__}, @@ -1847,6 +1866,32 @@ _curses_filter(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_CURSES_FILTER) */ +#if defined(HAVE_CURSES_NOFILTER) + +PyDoc_STRVAR(_curses_nofilter__doc__, +"nofilter($module, /)\n" +"--\n" +"\n" +"Undo the effect of a preceding filter() call.\n" +"\n" +"Must be called before initscr(). It restores the normal behaviour that\n" +"filter() disables, so that the next initscr() or newterm() uses the full\n" +"screen rather than a single line."); + +#define _CURSES_NOFILTER_METHODDEF \ + {"nofilter", (PyCFunction)_curses_nofilter, METH_NOARGS, _curses_nofilter__doc__}, + +static PyObject * +_curses_nofilter_impl(PyObject *module); + +static PyObject * +_curses_nofilter(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_nofilter_impl(module); +} + +#endif /* defined(HAVE_CURSES_NOFILTER) */ + PyDoc_STRVAR(_curses_baudrate__doc__, "baudrate($module, /)\n" "--\n" @@ -1910,11 +1955,12 @@ PyDoc_STRVAR(_curses_cbreak__doc__, " flag\n" " If false, the effect is the same as calling nocbreak().\n" "\n" -"In cbreak mode (sometimes called \"rare\" mode) normal tty line buffering is\n" -"turned off and characters are available to be read one by one. However,\n" -"unlike raw mode, special characters (interrupt, quit, suspend, and flow\n" -"control) retain their effects on the tty driver and calling program.\n" -"Calling first raw() then cbreak() leaves the terminal in cbreak mode."); +"In cbreak mode (sometimes called \"rare\" mode) normal tty line buffering\n" +"is turned off and characters are available to be read one by one.\n" +"However, unlike raw mode, special characters (interrupt, quit, suspend,\n" +"and flow control) retain their effects on the tty driver and calling\n" +"program. Calling first raw() then cbreak() leaves the terminal in\n" +"cbreak mode."); #define _CURSES_CBREAK_METHODDEF \ {"cbreak", _PyCFunction_CAST(_curses_cbreak), METH_FASTCALL, _curses_cbreak__doc__}, @@ -1945,6 +1991,94 @@ _curses_cbreak(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427) + +PyDoc_STRVAR(_curses_is_cbreak__doc__, +"is_cbreak($module, /)\n" +"--\n" +"\n" +"Return True if cbreak mode is enabled, False otherwise."); + +#define _CURSES_IS_CBREAK_METHODDEF \ + {"is_cbreak", (PyCFunction)_curses_is_cbreak, METH_NOARGS, _curses_is_cbreak__doc__}, + +static PyObject * +_curses_is_cbreak_impl(PyObject *module); + +static PyObject * +_curses_is_cbreak(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_is_cbreak_impl(module); +} + +#endif /* (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427) */ + +#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427) + +PyDoc_STRVAR(_curses_is_echo__doc__, +"is_echo($module, /)\n" +"--\n" +"\n" +"Return True if echo mode is enabled, False otherwise."); + +#define _CURSES_IS_ECHO_METHODDEF \ + {"is_echo", (PyCFunction)_curses_is_echo, METH_NOARGS, _curses_is_echo__doc__}, + +static PyObject * +_curses_is_echo_impl(PyObject *module); + +static PyObject * +_curses_is_echo(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_is_echo_impl(module); +} + +#endif /* (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427) */ + +#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427) + +PyDoc_STRVAR(_curses_is_nl__doc__, +"is_nl($module, /)\n" +"--\n" +"\n" +"Return True if nl mode is enabled, False otherwise."); + +#define _CURSES_IS_NL_METHODDEF \ + {"is_nl", (PyCFunction)_curses_is_nl, METH_NOARGS, _curses_is_nl__doc__}, + +static PyObject * +_curses_is_nl_impl(PyObject *module); + +static PyObject * +_curses_is_nl(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_is_nl_impl(module); +} + +#endif /* (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427) */ + +#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427) + +PyDoc_STRVAR(_curses_is_raw__doc__, +"is_raw($module, /)\n" +"--\n" +"\n" +"Return True if raw mode is enabled, False otherwise."); + +#define _CURSES_IS_RAW_METHODDEF \ + {"is_raw", (PyCFunction)_curses_is_raw, METH_NOARGS, _curses_is_raw__doc__}, + +static PyObject * +_curses_is_raw_impl(PyObject *module); + +static PyObject * +_curses_is_raw(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_is_raw_impl(module); +} + +#endif /* (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20240427) */ + PyDoc_STRVAR(_curses_color_content__doc__, "color_content($module, color_number, /)\n" "--\n" @@ -1954,8 +2088,9 @@ PyDoc_STRVAR(_curses_color_content__doc__, " color_number\n" " The number of the color (0 - (COLORS-1)).\n" "\n" -"A 3-tuple is returned, containing the R, G, B values for the given color,\n" -"which will be between 0 (no component) and 1000 (maximum amount of component)."); +"A 3-tuple is returned, containing the R, G, B values for the given\n" +"color, which will be between 0 (no component) and 1000 (maximum amount\n" +"of component)."); #define _CURSES_COLOR_CONTENT_METHODDEF \ {"color_content", (PyCFunction)_curses_color_content, METH_O, _curses_color_content__doc__}, @@ -1988,7 +2123,8 @@ PyDoc_STRVAR(_curses_color_pair__doc__, " The number of the color pair.\n" "\n" "This attribute value can be combined with A_STANDOUT, A_REVERSE, and the\n" -"other A_* attributes. pair_number() is the counterpart to this function."); +"other A_* attributes. pair_number() is the counterpart to this\n" +"function."); #define _CURSES_COLOR_PAIR_METHODDEF \ {"color_pair", (PyCFunction)_curses_color_pair, METH_O, _curses_color_pair__doc__}, @@ -2022,9 +2158,9 @@ PyDoc_STRVAR(_curses_curs_set__doc__, " 0 for invisible, 1 for normal visible, or 2 for very visible.\n" "\n" "If the terminal supports the visibility requested, the previous cursor\n" -"state is returned; otherwise, an exception is raised. On many terminals,\n" -"the \"visible\" mode is an underline cursor and the \"very visible\" mode is\n" -"a block cursor."); +"state is returned; otherwise, an exception is raised. On many\n" +"terminals, the \"visible\" mode is an underline cursor and the \"very\n" +"visible\" mode is a block cursor."); #define _CURSES_CURS_SET_METHODDEF \ {"curs_set", (PyCFunction)_curses_curs_set, METH_O, _curses_curs_set__doc__}, @@ -2076,7 +2212,8 @@ PyDoc_STRVAR(_curses_def_shell_mode__doc__, "\n" "Save the current terminal mode as the \"shell\" mode.\n" "\n" -"The \"shell\" mode is the mode when the running program is not using curses.\n" +"The \"shell\" mode is the mode when the running program is not using\n" +"curses.\n" "\n" "Subsequent calls to reset_shell_mode() will restore this mode."); @@ -2150,7 +2287,8 @@ PyDoc_STRVAR(_curses_echo__doc__, " flag\n" " If false, the effect is the same as calling noecho().\n" "\n" -"In echo mode, each character input is echoed to the screen as it is entered."); +"In echo mode, each character input is echoed to the screen as it is\n" +"entered."); #define _CURSES_ECHO_METHODDEF \ {"echo", _PyCFunction_CAST(_curses_echo), METH_FASTCALL, _curses_echo__doc__}, @@ -2217,13 +2355,36 @@ _curses_erasechar(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_erasechar_impl(module); } +#if defined(HAVE_NCURSESW) + +PyDoc_STRVAR(_curses_erasewchar__doc__, +"erasewchar($module, /)\n" +"--\n" +"\n" +"Return the user\'s current wide-character erase character."); + +#define _CURSES_ERASEWCHAR_METHODDEF \ + {"erasewchar", (PyCFunction)_curses_erasewchar, METH_NOARGS, _curses_erasewchar__doc__}, + +static PyObject * +_curses_erasewchar_impl(PyObject *module); + +static PyObject * +_curses_erasewchar(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_erasewchar_impl(module); +} + +#endif /* defined(HAVE_NCURSESW) */ + PyDoc_STRVAR(_curses_flash__doc__, "flash($module, /)\n" "--\n" "\n" "Flash the screen.\n" "\n" -"That is, change it to reverse-video and then change it back in a short interval."); +"That is, change it to reverse-video and then change it back in a short\n" +"interval."); #define _CURSES_FLASH_METHODDEF \ {"flash", (PyCFunction)_curses_flash, METH_NOARGS, _curses_flash__doc__}, @@ -2243,8 +2404,8 @@ PyDoc_STRVAR(_curses_flushinp__doc__, "\n" "Flush all input buffers.\n" "\n" -"This throws away any typeahead that has been typed by the user and has not\n" -"yet been processed by the program."); +"This throws away any typeahead that has been typed by the user and has\n" +"not yet been processed by the program."); #define _CURSES_FLUSHINP_METHODDEF \ {"flushinp", (PyCFunction)_curses_flushinp, METH_NOARGS, _curses_flushinp__doc__}, @@ -2614,8 +2775,9 @@ PyDoc_STRVAR(_curses_init_pair__doc__, " bg\n" " Background color number (-1 - (COLORS-1)).\n" "\n" -"If the color-pair was previously initialized, the screen is refreshed and\n" -"all occurrences of that color-pair are changed to the new definition."); +"If the color-pair was previously initialized, the screen is refreshed\n" +"and all occurrences of that color-pair are changed to the new\n" +"definition."); #define _CURSES_INIT_PAIR_METHODDEF \ {"init_pair", _PyCFunction_CAST(_curses_init_pair), METH_FASTCALL, _curses_init_pair__doc__}, @@ -2649,6 +2811,154 @@ _curses_init_pair(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#if (_NCURSES_EXTENDED_COLOR_FUNCS) + +PyDoc_STRVAR(_curses_alloc_pair__doc__, +"alloc_pair($module, fg, bg, /)\n" +"--\n" +"\n" +"Allocate a color pair for the given foreground and background colors.\n" +"\n" +" fg\n" +" Foreground color number.\n" +" bg\n" +" Background color number.\n" +"\n" +"If a color pair for the same colors already exists, return its number.\n" +"Otherwise allocate a new color pair and return its number."); + +#define _CURSES_ALLOC_PAIR_METHODDEF \ + {"alloc_pair", _PyCFunction_CAST(_curses_alloc_pair), METH_FASTCALL, _curses_alloc_pair__doc__}, + +static PyObject * +_curses_alloc_pair_impl(PyObject *module, int fg, int bg); + +static PyObject * +_curses_alloc_pair(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fg; + int bg; + + if (!_PyArg_CheckPositional("alloc_pair", nargs, 2, 2)) { + goto exit; + } + if (!color_allow_default_converter(args[0], &fg)) { + goto exit; + } + if (!color_allow_default_converter(args[1], &bg)) { + goto exit; + } + return_value = _curses_alloc_pair_impl(module, fg, bg); + +exit: + return return_value; +} + +#endif /* (_NCURSES_EXTENDED_COLOR_FUNCS) */ + +#if (_NCURSES_EXTENDED_COLOR_FUNCS) + +PyDoc_STRVAR(_curses_find_pair__doc__, +"find_pair($module, fg, bg, /)\n" +"--\n" +"\n" +"Return the number of a color pair for the given colors, or -1.\n" +"\n" +" fg\n" +" Foreground color number.\n" +" bg\n" +" Background color number.\n" +"\n" +"Return -1 if no color pair for this combination of foreground and\n" +"background colors has been allocated."); + +#define _CURSES_FIND_PAIR_METHODDEF \ + {"find_pair", _PyCFunction_CAST(_curses_find_pair), METH_FASTCALL, _curses_find_pair__doc__}, + +static PyObject * +_curses_find_pair_impl(PyObject *module, int fg, int bg); + +static PyObject * +_curses_find_pair(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int fg; + int bg; + + if (!_PyArg_CheckPositional("find_pair", nargs, 2, 2)) { + goto exit; + } + if (!color_allow_default_converter(args[0], &fg)) { + goto exit; + } + if (!color_allow_default_converter(args[1], &bg)) { + goto exit; + } + return_value = _curses_find_pair_impl(module, fg, bg); + +exit: + return return_value; +} + +#endif /* (_NCURSES_EXTENDED_COLOR_FUNCS) */ + +#if (_NCURSES_EXTENDED_COLOR_FUNCS) + +PyDoc_STRVAR(_curses_free_pair__doc__, +"free_pair($module, pair, /)\n" +"--\n" +"\n" +"Free a color pair allocated by alloc_pair().\n" +"\n" +" pair\n" +" The number of the color pair to free."); + +#define _CURSES_FREE_PAIR_METHODDEF \ + {"free_pair", (PyCFunction)_curses_free_pair, METH_O, _curses_free_pair__doc__}, + +static PyObject * +_curses_free_pair_impl(PyObject *module, int pair); + +static PyObject * +_curses_free_pair(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + int pair; + + if (!pair_converter(arg, &pair)) { + goto exit; + } + return_value = _curses_free_pair_impl(module, pair); + +exit: + return return_value; +} + +#endif /* (_NCURSES_EXTENDED_COLOR_FUNCS) */ + +#if (_NCURSES_EXTENDED_COLOR_FUNCS) + +PyDoc_STRVAR(_curses_reset_color_pairs__doc__, +"reset_color_pairs($module, /)\n" +"--\n" +"\n" +"Discard all color-pair definitions."); + +#define _CURSES_RESET_COLOR_PAIRS_METHODDEF \ + {"reset_color_pairs", (PyCFunction)_curses_reset_color_pairs, METH_NOARGS, _curses_reset_color_pairs__doc__}, + +static PyObject * +_curses_reset_color_pairs_impl(PyObject *module); + +static PyObject * +_curses_reset_color_pairs(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_reset_color_pairs_impl(module); +} + +#endif /* (_NCURSES_EXTENDED_COLOR_FUNCS) */ + PyDoc_STRVAR(_curses_initscr__doc__, "initscr($module, /)\n" "--\n" @@ -2766,6 +3076,112 @@ _curses_setupterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } +PyDoc_STRVAR(_curses_newterm__doc__, +"newterm($module, type=None, fd=None, infd=None, /)\n" +"--\n" +"\n" +"Return a new screen for the terminal, in addition to the initial screen.\n" +"\n" +" type\n" +" Terminal name; if None, the TERM environment variable is used.\n" +" fd\n" +" Output file object or descriptor (default: sys.stdout).\n" +" infd\n" +" Input file object or descriptor (default: sys.stdin).\n" +"\n" +"This is an alternative to initscr() for programs running on more than\n" +"one terminal. Use set_term() to switch between the screens."); + +#define _CURSES_NEWTERM_METHODDEF \ + {"newterm", _PyCFunction_CAST(_curses_newterm), METH_FASTCALL, _curses_newterm__doc__}, + +static PyObject * +_curses_newterm_impl(PyObject *module, const char *type, PyObject *fd, + PyObject *infd); + +static PyObject * +_curses_newterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + const char *type = NULL; + PyObject *fd = Py_None; + PyObject *infd = Py_None; + + if (!_PyArg_CheckPositional("newterm", nargs, 0, 3)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (args[0] == Py_None) { + type = NULL; + } + else if (PyUnicode_Check(args[0])) { + Py_ssize_t type_length; + type = PyUnicode_AsUTF8AndSize(args[0], &type_length); + if (type == NULL) { + goto exit; + } + if (strlen(type) != (size_t)type_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + } + else { + _PyArg_BadArgument("newterm", "argument 1", "str or None", args[0]); + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + fd = args[1]; + if (nargs < 3) { + goto skip_optional; + } + infd = args[2]; +skip_optional: + return_value = _curses_newterm_impl(module, type, fd, infd); + +exit: + return return_value; +} + +PyDoc_STRVAR(_curses_set_term__doc__, +"set_term($module, screen, /)\n" +"--\n" +"\n" +"Switch to the given screen and return the previously current screen.\n" +"\n" +"Returns None if the previous screen was the one created by initscr()."); + +#define _CURSES_SET_TERM_METHODDEF \ + {"set_term", (PyCFunction)_curses_set_term, METH_O, _curses_set_term__doc__}, + +#if defined(HAVE_CURSES_NEW_PRESCR) + +PyDoc_STRVAR(_curses_new_prescr__doc__, +"new_prescr($module, /)\n" +"--\n" +"\n" +"Create a screen and return it, without initializing a terminal.\n" +"\n" +"The screen can be used to call functions that affect the screen before\n" +"calling newterm() or initscr()."); + +#define _CURSES_NEW_PRESCR_METHODDEF \ + {"new_prescr", (PyCFunction)_curses_new_prescr, METH_NOARGS, _curses_new_prescr__doc__}, + +static PyObject * +_curses_new_prescr_impl(PyObject *module); + +static PyObject * +_curses_new_prescr(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_new_prescr_impl(module); +} + +#endif /* defined(HAVE_CURSES_NEW_PRESCR) */ + #if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) PyDoc_STRVAR(_curses_get_escdelay__doc__, @@ -2774,9 +3190,9 @@ PyDoc_STRVAR(_curses_get_escdelay__doc__, "\n" "Gets the curses ESCDELAY setting.\n" "\n" -"Gets the number of milliseconds to wait after reading an escape character,\n" -"to distinguish between an individual escape character entered on the\n" -"keyboard from escape sequences sent by cursor and function keys."); +"Gets the number of milliseconds to wait after reading an escape\n" +"character, to distinguish between an individual escape character entered\n" +"on the keyboard from escape sequences sent by cursor and function keys."); #define _CURSES_GET_ESCDELAY_METHODDEF \ {"get_escdelay", (PyCFunction)_curses_get_escdelay, METH_NOARGS, _curses_get_escdelay__doc__}, @@ -2803,9 +3219,9 @@ PyDoc_STRVAR(_curses_set_escdelay__doc__, " ms\n" " length of the delay in milliseconds.\n" "\n" -"Sets the number of milliseconds to wait after reading an escape character,\n" -"to distinguish between an individual escape character entered on the\n" -"keyboard from escape sequences sent by cursor and function keys."); +"Sets the number of milliseconds to wait after reading an escape\n" +"character, to distinguish between an individual escape character entered\n" +"on the keyboard from escape sequences sent by cursor and function keys."); #define _CURSES_SET_ESCDELAY_METHODDEF \ {"set_escdelay", (PyCFunction)_curses_set_escdelay, METH_O, _curses_set_escdelay__doc__}, @@ -2839,8 +3255,8 @@ PyDoc_STRVAR(_curses_get_tabsize__doc__, "\n" "Gets the curses TABSIZE setting.\n" "\n" -"Gets the number of columns used by the curses library when converting a tab\n" -"character to spaces as it adds the tab to a window."); +"Gets the number of columns used by the curses library when converting\n" +"a tab character to spaces as it adds the tab to a window."); #define _CURSES_GET_TABSIZE_METHODDEF \ {"get_tabsize", (PyCFunction)_curses_get_tabsize, METH_NOARGS, _curses_get_tabsize__doc__}, @@ -2867,8 +3283,8 @@ PyDoc_STRVAR(_curses_set_tabsize__doc__, " size\n" " rendered cell width of a tab character.\n" "\n" -"Sets the number of columns used by the curses library when converting a tab\n" -"character to spaces as it adds the tab to a window."); +"Sets the number of columns used by the curses library when converting\n" +"a tab character to spaces as it adds the tab to a window."); #define _CURSES_SET_TABSIZE_METHODDEF \ {"set_tabsize", (PyCFunction)_curses_set_tabsize, METH_O, _curses_set_tabsize__doc__}, @@ -2897,7 +3313,12 @@ _curses_set_tabsize(PyObject *module, PyObject *arg) PyDoc_STRVAR(_curses_intrflush__doc__, "intrflush($module, flag, /)\n" "--\n" -"\n"); +"\n" +"Control flushing of the output buffer when an interrupt key is pressed.\n" +"\n" +"If flag is true, pressing an interrupt key (interrupt, break, or quit)\n" +"flushes all output in the terminal driver queue. If flag is false, no\n" +"flushing is done."); #define _CURSES_INTRFLUSH_METHODDEF \ {"intrflush", (PyCFunction)_curses_intrflush, METH_O, _curses_intrflush__doc__}, @@ -3033,14 +3454,36 @@ _curses_killchar(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_killchar_impl(module); } +#if defined(HAVE_NCURSESW) + +PyDoc_STRVAR(_curses_killwchar__doc__, +"killwchar($module, /)\n" +"--\n" +"\n" +"Return the user\'s current wide-character line kill character."); + +#define _CURSES_KILLWCHAR_METHODDEF \ + {"killwchar", (PyCFunction)_curses_killwchar, METH_NOARGS, _curses_killwchar__doc__}, + +static PyObject * +_curses_killwchar_impl(PyObject *module); + +static PyObject * +_curses_killwchar(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _curses_killwchar_impl(module); +} + +#endif /* defined(HAVE_NCURSESW) */ + PyDoc_STRVAR(_curses_longname__doc__, "longname($module, /)\n" "--\n" "\n" "Return the terminfo long name field describing the current terminal.\n" "\n" -"The maximum length of a verbose description is 128 characters. It is defined\n" -"only after the call to initscr()."); +"The maximum length of a verbose description is 128 characters. It is\n" +"defined only after the call to initscr()."); #define _CURSES_LONGNAME_METHODDEF \ {"longname", (PyCFunction)_curses_longname, METH_NOARGS, _curses_longname__doc__}, @@ -3097,8 +3540,8 @@ PyDoc_STRVAR(_curses_mouseinterval__doc__, " Time in milliseconds.\n" "\n" "Set the maximum time that can elapse between press and release events in\n" -"order for them to be recognized as a click, and return the previous interval\n" -"value."); +"order for them to be recognized as a click, and return the previous\n" +"interval value."); #define _CURSES_MOUSEINTERVAL_METHODDEF \ {"mouseinterval", (PyCFunction)_curses_mouseinterval, METH_O, _curses_mouseinterval__doc__}, @@ -3130,12 +3573,12 @@ PyDoc_STRVAR(_curses_mousemask__doc__, "mousemask($module, newmask, /)\n" "--\n" "\n" -"Set the mouse events to be reported, and return a tuple (availmask, oldmask).\n" +"Set the mouse events to be reported, and return (availmask, oldmask).\n" "\n" "Return a tuple (availmask, oldmask). availmask indicates which of the\n" -"specified mouse events can be reported; on complete failure it returns 0.\n" -"oldmask is the previous value of the given window\'s mouse event mask.\n" -"If this function is never called, no mouse events are ever reported."); +"specified mouse events can be reported; on complete failure it returns\n" +"0. oldmask is the previous value of the mouse event mask. If this\n" +"function is never called, no mouse events are ever reported."); #define _CURSES_MOUSEMASK_METHODDEF \ {"mousemask", (PyCFunction)_curses_mousemask, METH_O, _curses_mousemask__doc__}, @@ -3267,8 +3710,8 @@ PyDoc_STRVAR(_curses_newwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"By default, the window will extend from the specified position to the lower\n" -"right corner of the screen."); +"By default, the window will extend from the specified position to the\n" +"lower right corner of the screen."); #define _CURSES_NEWWIN_METHODDEF \ {"newwin", (PyCFunction)_curses_newwin, METH_VARARGS, _curses_newwin__doc__}, @@ -3318,8 +3761,9 @@ PyDoc_STRVAR(_curses_nl__doc__, " flag\n" " If false, the effect is the same as calling nonl().\n" "\n" -"This mode translates the return key into newline on input, and translates\n" -"newline into return and line-feed on output. Newline mode is initially on."); +"This mode translates the return key into newline on input, and\n" +"translates newline into return and line-feed on output. Newline mode\n" +"is initially on."); #define _CURSES_NL_METHODDEF \ {"nl", _PyCFunction_CAST(_curses_nl), METH_FASTCALL, _curses_nl__doc__}, @@ -3396,8 +3840,8 @@ PyDoc_STRVAR(_curses_nonl__doc__, "\n" "Leave newline mode.\n" "\n" -"Disable translation of return into newline on input, and disable low-level\n" -"translation of newline into newline/return on output."); +"Disable translation of return into newline on input, and disable\n" +"low-level translation of newline into newline/return on output."); #define _CURSES_NONL_METHODDEF \ {"nonl", (PyCFunction)_curses_nonl, METH_NOARGS, _curses_nonl__doc__}, @@ -3587,7 +4031,10 @@ _curses_qiflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(_curses_update_lines_cols__doc__, "update_lines_cols($module, /)\n" "--\n" -"\n"); +"\n" +"Update the LINES and COLS module variables.\n" +"\n" +"This is useful for detecting manual screen resize."); #define _CURSES_UPDATE_LINES_COLS_METHODDEF \ {"update_lines_cols", (PyCFunction)_curses_update_lines_cols, METH_NOARGS, _curses_update_lines_cols__doc__}, @@ -3613,8 +4060,8 @@ PyDoc_STRVAR(_curses_raw__doc__, " If false, the effect is the same as calling noraw().\n" "\n" "In raw mode, normal line buffering and processing of interrupt, quit,\n" -"suspend, and flow control keys are turned off; characters are presented to\n" -"curses input functions one by one."); +"suspend, and flow control keys are turned off; characters are presented\n" +"to curses input functions one by one."); #define _CURSES_RAW_METHODDEF \ {"raw", _PyCFunction_CAST(_curses_raw), METH_FASTCALL, _curses_raw__doc__}, @@ -3712,8 +4159,8 @@ PyDoc_STRVAR(_curses_resizeterm__doc__, " ncols\n" " Width.\n" "\n" -"Adjusts other bookkeeping data used by the curses library that record the\n" -"window dimensions (in particular the SIGWINCH handler)."); +"Adjusts other bookkeeping data used by the curses library that record\n" +"the window dimensions (in particular the SIGWINCH handler)."); #define _CURSES_RESIZETERM_METHODDEF \ {"resizeterm", _PyCFunction_CAST(_curses_resizeterm), METH_FASTCALL, _curses_resizeterm__doc__}, @@ -3791,10 +4238,11 @@ PyDoc_STRVAR(_curses_resize_term__doc__, " Width.\n" "\n" "When resizing the windows, resize_term() blank-fills the areas that are\n" -"extended. The calling application should fill in these areas with appropriate\n" -"data. The resize_term() function attempts to resize all windows. However,\n" -"due to the calling convention of pads, it is not possible to resize these\n" -"without additional interaction with the application."); +"extended. The calling application should fill in these areas with\n" +"appropriate data. The resize_term() function attempts to resize all\n" +"windows. However, due to the calling convention of pads, it is not\n" +"possible to resize these without additional interaction with the\n" +"application."); #define _CURSES_RESIZE_TERM_METHODDEF \ {"resize_term", _PyCFunction_CAST(_curses_resize_term), METH_FASTCALL, _curses_resize_term__doc__}, @@ -3929,12 +4377,12 @@ PyDoc_STRVAR(_curses_start_color__doc__, "\n" "Initializes eight basic colors and global variables COLORS and COLOR_PAIRS.\n" "\n" -"Must be called if the programmer wants to use colors, and before any other\n" -"color manipulation routine is called. It is good practice to call this\n" -"routine right after initscr().\n" +"Must be called if the programmer wants to use colors, and before any\n" +"other color manipulation routine is called. It is good practice to call\n" +"this routine right after initscr().\n" "\n" -"It also restores the colors on the terminal to the values they had when the\n" -"terminal was just turned on."); +"It also restores the colors on the terminal to the values they had when\n" +"the terminal was just turned on."); #define _CURSES_START_COLOR_METHODDEF \ {"start_color", (PyCFunction)_curses_start_color, METH_NOARGS, _curses_start_color__doc__}, @@ -4036,8 +4484,8 @@ PyDoc_STRVAR(_curses_tigetnum__doc__, " capname\n" " The terminfo capability name.\n" "\n" -"The value -2 is returned if capname is not a numeric capability, or -1 if\n" -"it is canceled or absent from the terminal description."); +"The value -2 is returned if capname is not a numeric capability, or -1\n" +"if it is canceled or absent from the terminal description."); #define _CURSES_TIGETNUM_METHODDEF \ {"tigetnum", (PyCFunction)_curses_tigetnum, METH_O, _curses_tigetnum__doc__}, @@ -4079,8 +4527,8 @@ PyDoc_STRVAR(_curses_tigetstr__doc__, " capname\n" " The terminfo capability name.\n" "\n" -"None is returned if capname is not a string capability, or is canceled or\n" -"absent from the terminal description."); +"None is returned if capname is not a string capability, or is canceled\n" +"or absent from the terminal description."); #define _CURSES_TIGETSTR_METHODDEF \ {"tigetstr", (PyCFunction)_curses_tigetstr, METH_O, _curses_tigetstr__doc__}, @@ -4204,6 +4652,22 @@ PyDoc_STRVAR(_curses_unctrl__doc__, #define _CURSES_UNCTRL_METHODDEF \ {"unctrl", (PyCFunction)_curses_unctrl, METH_O, _curses_unctrl__doc__}, +#if defined(HAVE_NCURSESW) + +PyDoc_STRVAR(_curses_wunctrl__doc__, +"wunctrl($module, ch, /)\n" +"--\n" +"\n" +"Return a printable representation of the wide character ch.\n" +"\n" +"Control characters are displayed as a caret followed by the character,\n" +"for example as ^C. Printing characters are left as they are."); + +#define _CURSES_WUNCTRL_METHODDEF \ + {"wunctrl", (PyCFunction)_curses_wunctrl, METH_O, _curses_wunctrl__doc__}, + +#endif /* defined(HAVE_NCURSESW) */ + PyDoc_STRVAR(_curses_ungetch__doc__, "ungetch($module, ch, /)\n" "--\n" @@ -4234,14 +4698,14 @@ PyDoc_STRVAR(_curses_use_env__doc__, "\n" "Use environment variables LINES and COLUMNS.\n" "\n" -"If used, this function should be called before initscr() or newterm() are\n" -"called.\n" +"If used, this function should be called before initscr() or newterm()\n" +"are called.\n" "\n" -"When flag is False, the values of lines and columns specified in the terminfo\n" -"database will be used, even if environment variables LINES and COLUMNS (used\n" -"by default) are set, or if curses is running in a window (in which case\n" -"default behavior would be to use the window size if LINES and COLUMNS are\n" -"not set)."); +"When flag is False, the values of lines and columns specified in the\n" +"terminfo database will be used, even if environment variables LINES and\n" +"COLUMNS (used by default) are set, or if curses is running in a window\n" +"(in which case default behavior would be to use the window size if LINES\n" +"and COLUMNS are not set)."); #define _CURSES_USE_ENV_METHODDEF \ {"use_env", (PyCFunction)_curses_use_env, METH_O, _curses_use_env__doc__}, @@ -4371,6 +4835,30 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #define _CURSES_FILTER_METHODDEF #endif /* !defined(_CURSES_FILTER_METHODDEF) */ +#ifndef _CURSES_NOFILTER_METHODDEF + #define _CURSES_NOFILTER_METHODDEF +#endif /* !defined(_CURSES_NOFILTER_METHODDEF) */ + +#ifndef _CURSES_IS_CBREAK_METHODDEF + #define _CURSES_IS_CBREAK_METHODDEF +#endif /* !defined(_CURSES_IS_CBREAK_METHODDEF) */ + +#ifndef _CURSES_IS_ECHO_METHODDEF + #define _CURSES_IS_ECHO_METHODDEF +#endif /* !defined(_CURSES_IS_ECHO_METHODDEF) */ + +#ifndef _CURSES_IS_NL_METHODDEF + #define _CURSES_IS_NL_METHODDEF +#endif /* !defined(_CURSES_IS_NL_METHODDEF) */ + +#ifndef _CURSES_IS_RAW_METHODDEF + #define _CURSES_IS_RAW_METHODDEF +#endif /* !defined(_CURSES_IS_RAW_METHODDEF) */ + +#ifndef _CURSES_ERASEWCHAR_METHODDEF + #define _CURSES_ERASEWCHAR_METHODDEF +#endif /* !defined(_CURSES_ERASEWCHAR_METHODDEF) */ + #ifndef _CURSES_GETSYX_METHODDEF #define _CURSES_GETSYX_METHODDEF #endif /* !defined(_CURSES_GETSYX_METHODDEF) */ @@ -4387,6 +4875,26 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #define _CURSES_HAS_KEY_METHODDEF #endif /* !defined(_CURSES_HAS_KEY_METHODDEF) */ +#ifndef _CURSES_ALLOC_PAIR_METHODDEF + #define _CURSES_ALLOC_PAIR_METHODDEF +#endif /* !defined(_CURSES_ALLOC_PAIR_METHODDEF) */ + +#ifndef _CURSES_FIND_PAIR_METHODDEF + #define _CURSES_FIND_PAIR_METHODDEF +#endif /* !defined(_CURSES_FIND_PAIR_METHODDEF) */ + +#ifndef _CURSES_FREE_PAIR_METHODDEF + #define _CURSES_FREE_PAIR_METHODDEF +#endif /* !defined(_CURSES_FREE_PAIR_METHODDEF) */ + +#ifndef _CURSES_RESET_COLOR_PAIRS_METHODDEF + #define _CURSES_RESET_COLOR_PAIRS_METHODDEF +#endif /* !defined(_CURSES_RESET_COLOR_PAIRS_METHODDEF) */ + +#ifndef _CURSES_NEW_PRESCR_METHODDEF + #define _CURSES_NEW_PRESCR_METHODDEF +#endif /* !defined(_CURSES_NEW_PRESCR_METHODDEF) */ + #ifndef _CURSES_GET_ESCDELAY_METHODDEF #define _CURSES_GET_ESCDELAY_METHODDEF #endif /* !defined(_CURSES_GET_ESCDELAY_METHODDEF) */ @@ -4407,6 +4915,10 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #define _CURSES_IS_TERM_RESIZED_METHODDEF #endif /* !defined(_CURSES_IS_TERM_RESIZED_METHODDEF) */ +#ifndef _CURSES_KILLWCHAR_METHODDEF + #define _CURSES_KILLWCHAR_METHODDEF +#endif /* !defined(_CURSES_KILLWCHAR_METHODDEF) */ + #ifndef _CURSES_MOUSEINTERVAL_METHODDEF #define _CURSES_MOUSEINTERVAL_METHODDEF #endif /* !defined(_CURSES_MOUSEINTERVAL_METHODDEF) */ @@ -4435,6 +4947,10 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #define _CURSES_TYPEAHEAD_METHODDEF #endif /* !defined(_CURSES_TYPEAHEAD_METHODDEF) */ +#ifndef _CURSES_WUNCTRL_METHODDEF + #define _CURSES_WUNCTRL_METHODDEF +#endif /* !defined(_CURSES_WUNCTRL_METHODDEF) */ + #ifndef _CURSES_UNGET_WCH_METHODDEF #define _CURSES_UNGET_WCH_METHODDEF #endif /* !defined(_CURSES_UNGET_WCH_METHODDEF) */ @@ -4450,4 +4966,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=135246e29163510c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fd0f4e65dc594a65 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index ee621c150c31e46..fac41e7aefc7f4f 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -214,8 +214,8 @@ PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "\n" "Create a date from a POSIX timestamp.\n" "\n" -"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" -"as local time."); +"The timestamp is a number, e.g. created via time.time(), that is\n" +"interpreted as local time."); #define DATETIME_DATE_FROMTIMESTAMP_METHODDEF \ {"fromtimestamp", (PyCFunction)datetime_date_fromtimestamp, METH_O|METH_CLASS, datetime_date_fromtimestamp__doc__}, @@ -897,8 +897,8 @@ PyDoc_STRVAR(datetime_time_isoformat__doc__, "\n" "Return the time formatted according to ISO.\n" "\n" -"The full format is \'HH:MM:SS.mmmmmm+zz:zz\'. By default, the fractional\n" -"part is omitted if self.microsecond == 0.\n" +"The full format is \'HH:MM:SS.mmmmmm+zz:zz\'. By default, the\n" +"fractional part is omitted if self.microsecond == 0.\n" "\n" "The optional argument timespec specifies the number of additional\n" "terms of the time to include. Valid options are \'auto\', \'hours\',\n" @@ -979,7 +979,8 @@ PyDoc_STRVAR(datetime_time_strftime__doc__, "\n" "Format using strftime().\n" "\n" -"The date part of the timestamp passed to underlying strftime should not be used.\n" +"The date part of the timestamp passed to underlying strftime should\n" +"not be used.\n" "\n" "For a list of supported format codes, see the documentation:\n" " https://docs.python.org/3/library/datetime.html#format-codes"); @@ -1269,8 +1270,8 @@ PyDoc_STRVAR(datetime_datetime__doc__, "\n" "A combination of a date and a time.\n" "\n" -"The year, month and day arguments are required. tzinfo may be None, or an\n" -"instance of a tzinfo subclass. The remaining arguments may be ints."); +"The year, month and day arguments are required. tzinfo may be None, or\n" +"an instance of a tzinfo subclass. The remaining arguments may be ints."); static PyObject * datetime_datetime_impl(PyTypeObject *type, int year, int month, int day, @@ -1491,8 +1492,8 @@ PyDoc_STRVAR(datetime_datetime_fromtimestamp__doc__, "\n" "Create a datetime from a POSIX timestamp.\n" "\n" -"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" -"as local time."); +"The timestamp is a number, e.g. created via time.time(), that is\n" +"interpreted as local time."); #define DATETIME_DATETIME_FROMTIMESTAMP_METHODDEF \ {"fromtimestamp", _PyCFunction_CAST(datetime_datetime_fromtimestamp), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, datetime_datetime_fromtimestamp__doc__}, @@ -2090,4 +2091,4 @@ datetime_datetime___reduce__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return datetime_datetime___reduce___impl((PyDateTime_DateTime *)self); } -/*[clinic end generated code: output=69658acff6a43ac4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8f63509398651723 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 091ce9edc43d4bd..6c979a4b0081df3 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -113,7 +113,8 @@ PyDoc_STRVAR(_dbm_dbm_setdefault__doc__, "\n" "Return the value for key if present, otherwise default.\n" "\n" -"If key is not in the database, it is inserted with default as the value."); +"If key is not in the database, it is inserted with default as the\n" +"value."); #define _DBM_DBM_SETDEFAULT_METHODDEF \ {"setdefault", _PyCFunction_CAST(_dbm_dbm_setdefault), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _dbm_dbm_setdefault__doc__}, @@ -246,4 +247,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=279511ea7cda38dd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=677deecf525167a5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index 23f666310850312..5f350c864f057b0 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -71,14 +71,15 @@ _functools_cmp_to_key(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } PyDoc_STRVAR(_functools_reduce__doc__, -"reduce($module, function, iterable, /, initial=<unrepresentable>)\n" +"reduce($module, function, iterable, /,\n" +" initial=functools._initial_missing)\n" "--\n" "\n" "Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" "\n" -"This effectively reduces the iterable to a single value. If initial is present,\n" -"it is placed before the items of the iterable in the calculation, and serves as\n" -"a default when the iterable is empty.\n" +"This effectively reduces the iterable to a single value. If initial is\n" +"present, it is placed before the items of the iterable in the\n" +"calculation, and serves as a default when the iterable is empty.\n" "\n" "For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n" "calculates ((((1 + 2) + 3) + 4) + 5)."); @@ -192,4 +193,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=7f2abc718fcc35d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d8fdaeba4b520fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 6fd6aa3da503351..fe993cc328fbd2f 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -138,9 +138,9 @@ PyDoc_STRVAR(_gdbm_gdbm_firstkey__doc__, "\n" "Return the starting key for the traversal.\n" "\n" -"It\'s possible to loop over every key in the database using this method\n" -"and the nextkey() method. The traversal is ordered by GDBM\'s internal\n" -"hash values, and won\'t be sorted by the key values."); +"It\'s possible to loop over every key in the database using this\n" +"method and the nextkey() method. The traversal is ordered by GDBM\'s\n" +"internal hash values, and won\'t be sorted by the key values."); #define _GDBM_GDBM_FIRSTKEY_METHODDEF \ {"firstkey", _PyCFunction_CAST(_gdbm_gdbm_firstkey), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_firstkey__doc__}, @@ -171,8 +171,8 @@ PyDoc_STRVAR(_gdbm_gdbm_nextkey__doc__, "\n" "Returns the key that follows key in the traversal.\n" "\n" -"The following code prints every key in the database db, without having\n" -"to create a list in memory that contains them all:\n" +"The following code prints every key in the database db, without\n" +"having to create a list in memory that contains them all:\n" "\n" " k = db.firstkey()\n" " while k is not None:\n" @@ -226,9 +226,9 @@ PyDoc_STRVAR(_gdbm_gdbm_reorganize__doc__, "\n" "If you have carried out a lot of deletions and would like to shrink\n" "the space used by the GDBM file, this routine will reorganize the\n" -"database. GDBM will not shorten the length of a database file except\n" -"by using this reorganization; otherwise, deleted file space will be\n" -"kept and reused as new (key,value) pairs are added."); +"database. GDBM will not shorten the length of a database file\n" +"except by using this reorganization; otherwise, deleted file space\n" +"will be kept and reused as new (key,value) pairs are added."); #define _GDBM_GDBM_REORGANIZE_METHODDEF \ {"reorganize", _PyCFunction_CAST(_gdbm_gdbm_reorganize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_reorganize__doc__}, @@ -389,4 +389,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=8bca34ce9d4493dd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=429b5db24568292e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 7ae7be185eceb58..17ed37bbd3b6191 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -1865,8 +1865,8 @@ PyDoc_STRVAR(_hashlib_HMAC_hexdigest__doc__, "\n" "Return hexadecimal digest of the bytes passed to the update() method so far.\n" "\n" -"This may be used to exchange the value safely in email or other non-binary\n" -"environments."); +"This may be used to exchange the value safely in email or other\n" +"non-binary environments."); #define _HASHLIB_HMAC_HEXDIGEST_METHODDEF \ {"hexdigest", (PyCFunction)_hashlib_HMAC_hexdigest, METH_NOARGS, _hashlib_HMAC_hexdigest__doc__}, @@ -1887,8 +1887,8 @@ PyDoc_STRVAR(_hashlib_get_fips_mode__doc__, "Determine the OpenSSL FIPS mode of operation.\n" "\n" "For OpenSSL 3.0.0 and newer it returns the state of the default provider\n" -"in the default OSSL context. It\'s not quite the same as FIPS_mode() but good\n" -"enough for unittests.\n" +"in the default OSSL context. It\'s not quite the same as FIPS_mode() but\n" +"good enough for unittests.\n" "\n" "Effectively any non-zero return value indicates FIPS mode;\n" "values other than 1 may have additional significance."); @@ -1986,4 +1986,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #define _HASHLIB_OPENSSL_SHAKE_256_METHODDEF #endif /* !defined(_HASHLIB_OPENSSL_SHAKE_256_METHODDEF) */ -/*[clinic end generated code: output=9ba35fcc33795b1e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cf405e652a340bb2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_heapqmodule.c.h b/Modules/clinic/_heapqmodule.c.h index b43155b6c24e3ce..f3b8256efc02216 100644 --- a/Modules/clinic/_heapqmodule.c.h +++ b/Modules/clinic/_heapqmodule.c.h @@ -326,8 +326,8 @@ PyDoc_STRVAR(_heapq_heappushpop_max__doc__, "\n" "Maxheap variant of heappushpop.\n" "\n" -"The combined action runs more efficiently than heappush_max() followed by\n" -"a separate call to heappop_max()."); +"The combined action runs more efficiently than heappush_max()\n" +"followed by a separate call to heappop_max()."); #define _HEAPQ_HEAPPUSHPOP_MAX_METHODDEF \ {"heappushpop_max", _PyCFunction_CAST(_heapq_heappushpop_max), METH_FASTCALL, _heapq_heappushpop_max__doc__}, @@ -358,4 +358,4 @@ _heapq_heappushpop_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=e83d50002c29a96d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=21e4f248ef6e83d6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpqueuesmodule.c.h b/Modules/clinic/_interpqueuesmodule.c.h index 3f08a0cb6d36ab1..6f27ea73b8866fe 100644 --- a/Modules/clinic/_interpqueuesmodule.c.h +++ b/Modules/clinic/_interpqueuesmodule.c.h @@ -109,7 +109,8 @@ PyDoc_STRVAR(_interpqueues_destroy__doc__, "\n" "Clear and destroy the queue.\n" "\n" -"Afterward attempts to use the queue will behave as though it never existed."); +"Afterward attempts to use the queue will behave as though it never\n" +"existed."); #define _INTERPQUEUES_DESTROY_METHODDEF \ {"destroy", _PyCFunction_CAST(_interpqueues_destroy), METH_FASTCALL|METH_KEYWORDS, _interpqueues_destroy__doc__}, @@ -762,4 +763,4 @@ _interpqueues__register_heap_types(PyObject *module, PyObject *const *args, Py_s exit: return return_value; } -/*[clinic end generated code: output=64cea8e1063429b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7e56e5b0c684d294 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_interpretersmodule.c.h b/Modules/clinic/_interpretersmodule.c.h index d70ffcea527895a..72792f9b583a66c 100644 --- a/Modules/clinic/_interpretersmodule.c.h +++ b/Modules/clinic/_interpretersmodule.c.h @@ -541,7 +541,8 @@ PyDoc_STRVAR(_interpreters_run_func__doc__, "Execute the body of the provided function in the identified interpreter.\n" "\n" "Code objects are also supported. In both cases, closures and args\n" -"are not supported. Methods and other callables are not supported either.\n" +"are not supported. Methods and other callables are not supported\n" +"either.\n" "\n" "(See _interpreters.exec().)"); @@ -1139,8 +1140,9 @@ PyDoc_STRVAR(_interpreters_capture_exception__doc__, "\n" "Return a snapshot of an exception.\n" "\n" -"If \"exc\" is None then the current exception, if any, is used (but not cleared).\n" -"The returned snapshot is the same as what _interpreters.exec() returns."); +"If \"exc\" is None then the current exception, if any, is used (but not\n" +"cleared). The returned snapshot is the same as what\n" +"_interpreters.exec() returns."); #define _INTERPRETERS_CAPTURE_EXCEPTION_METHODDEF \ {"capture_exception", _PyCFunction_CAST(_interpreters_capture_exception), METH_FASTCALL|METH_KEYWORDS, _interpreters_capture_exception__doc__}, @@ -1198,4 +1200,4 @@ _interpreters_capture_exception(PyObject *module, PyObject *const *args, Py_ssiz exit: return return_value; } -/*[clinic end generated code: output=c80f73761f860f6c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8c3ca09c304378ad input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_json.c.h b/Modules/clinic/_json.c.h index cd37a236c7611a1..d3ed1dfe0515a0a 100644 --- a/Modules/clinic/_json.c.h +++ b/Modules/clinic/_json.c.h @@ -16,8 +16,8 @@ PyDoc_STRVAR(py_scanstring__doc__, "ValueError on attempt to decode an invalid string. If strict is False\n" "then literal control characters are allowed in the string.\n" "\n" -"Returns a tuple of the decoded string and the index of the character in s\n" -"after the end quote."); +"Returns a tuple of the decoded string and the index of the character in\n" +"s after the end quote."); #define PY_SCANSTRING_METHODDEF \ {"scanstring", _PyCFunction_CAST(py_scanstring), METH_FASTCALL, py_scanstring__doc__}, @@ -125,4 +125,4 @@ py_encode_basestring(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=5bdd16375c95a4d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ea6e9a271d4ceaf2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index ebdc81a0dac2f05..bba107e8f806daf 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -74,18 +74,19 @@ PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_lzma_LZMADecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _lzma_LZMADecompressor_decompress__doc__}, @@ -333,4 +334,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=6386084cb43d2533 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ffc6d673d858048c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index d52629cf6eaa5b7..e7e9707f182a2ed 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -14,15 +14,15 @@ PyDoc_STRVAR(subprocess_fork_exec__doc__, "\n" "Spawn a fresh new child process.\n" "\n" -"Fork a child process, close parent file descriptors as appropriate in the\n" -"child and duplicate the few that are needed before calling exec() in the\n" -"child process.\n" +"Fork a child process, close parent file descriptors as appropriate in\n" +"the child and duplicate the few that are needed before calling exec() in\n" +"the child process.\n" "\n" -"If close_fds is True, close file descriptors 3 and higher, except those listed\n" -"in the sorted tuple pass_fds.\n" +"If close_fds is True, close file descriptors 3 and higher, except those\n" +"listed in the sorted tuple pass_fds.\n" "\n" -"The preexec_fn, if supplied, will be called immediately before closing file\n" -"descriptors and exec.\n" +"The preexec_fn, if supplied, will be called immediately before closing\n" +"file descriptors and exec.\n" "\n" "WARNING: preexec_fn is NOT SAFE if your application uses threads.\n" " It may trigger infrequent, difficult to debug deadlocks.\n" @@ -150,4 +150,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=942bc2748a9c2785 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=138941c284792aa1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index c9482f40acb9d4c..b67dd23f260c9e2 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -44,8 +44,9 @@ PyDoc_STRVAR(_queue_SimpleQueue_put__doc__, "\n" "Put the item on the queue.\n" "\n" -"The optional \'block\' and \'timeout\' arguments are ignored, as this method\n" -"never blocks. They are provided for compatibility with the Queue class."); +"The optional \'block\' and \'timeout\' arguments are ignored, as this\n" +"method never blocks. They are provided for compatibility with the\n" +"Queue class."); #define _QUEUE_SIMPLEQUEUE_PUT_METHODDEF \ {"put", _PyCFunction_CAST(_queue_SimpleQueue_put), METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_put__doc__}, @@ -188,10 +189,11 @@ PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, "\n" "Remove and return an item from the queue.\n" "\n" -"If optional args \'block\' is true and \'timeout\' is None (the default),\n" -"block if necessary until an item is available. If \'timeout\' is\n" -"a non-negative number, it blocks at most \'timeout\' seconds and raises\n" -"the Empty exception if no item was available within that time.\n" +"If optional args \'block\' is true and \'timeout\' is None (the\n" +"default), block if necessary until an item is available. If\n" +"\'timeout\' is a non-negative number, it blocks at most \'timeout\'\n" +"seconds and raises the Empty exception if no item was available\n" +"within that time.\n" "Otherwise (\'block\' is false), return an item if one is immediately\n" "available, else raise the Empty exception (\'timeout\' is ignored\n" "in that case)."); @@ -388,4 +390,4 @@ _queue_SimpleQueue___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=4af5d1b1ea31ac7d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8219fe2f2ed5f068 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 2563a16aea0b6f9..ca9cad7a572dadf 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -143,4 +143,35 @@ _random_Random_getrandbits(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=7ce97b2194eecaf7 input=a9049054013a1b77]*/ + +static int +random_init_impl(RandomObject *self, PyObject *seed); + +static int +random_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = (PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type; + PyObject *seed = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Random", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Random", PyTuple_GET_SIZE(args), 0, 1)) { + goto exit; + } + if (PyTuple_GET_SIZE(args) < 1) { + goto skip_optional; + } + seed = PyTuple_GET_ITEM(args, 0); +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = random_init_impl((RandomObject *)self, seed); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} +/*[clinic end generated code: output=ec95f7df0c3f3c19 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 8c35c8443b775ae..e337ed2390a1fc4 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -334,9 +334,10 @@ _ssl__SSLSocket_compression(PyObject *self, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, "This changes the context associated with the SSLSocket.\n" "\n" -"This is typically used from within a callback function set by the sni_callback\n" -"on the SSLContext to change the certificate information associated with the\n" -"SSLSocket before the cryptographic exchange handshake messages."); +"This is typically used from within a callback function set by the\n" +"sni_callback on the SSLContext to change the certificate information\n" +"associated with the SSLSocket before the cryptographic exchange\n" +"handshake messages."); #if defined(_ssl__SSLSocket_context_DOCSTR) # undef _ssl__SSLSocket_context_DOCSTR #endif @@ -571,9 +572,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_sendfile__doc__, "\n" "Write size bytes from offset in the file descriptor fd to the SSL connection.\n" "\n" -"This method uses the zero-copy technique and returns the number of bytes\n" -"written. It should be called only when Kernel TLS is used for sending data in\n" -"the connection.\n" +"This method uses the zero-copy technique and returns the number of\n" +"bytes written. It should be called only when Kernel TLS is used for\n" +"sending data in the connection.\n" "\n" "The meaning of flags is platform dependent."); @@ -762,8 +763,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_get_channel_binding__doc__, "\n" "Get channel binding data for current connection.\n" "\n" -"Raise ValueError if the requested `cb_type` is not supported. Return bytes\n" -"of the data or None if the data is not available (e.g. before the handshake).\n" +"Raise ValueError if the requested `cb_type` is not supported.\n" +"Return bytes of the data or None if the data is not available (e.g.\n" +"before the handshake).\n" "Only \'tls-unique\' channel binding data from RFC 5929 is supported."); #define _SSL__SSLSOCKET_GET_CHANNEL_BINDING_METHODDEF \ @@ -2210,8 +2212,9 @@ _ssl__SSLContext_set_ecdh_curve(PyObject *self, PyObject *name) PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n" "\n" -"If the argument is None then the callback is disabled. The method is called\n" -"with the SSLSocket, the server name as a string, and the SSLContext object.\n" +"If the argument is None then the callback is disabled. The method\n" +"is called with the SSLSocket, the server name as a string, and the\n" +"SSLContext object.\n" "\n" "See RFC 6066 for details of the SNI extension."); #if defined(_ssl__SSLContext_sni_callback_DOCSTR) @@ -2275,11 +2278,11 @@ PyDoc_STRVAR(_ssl__SSLContext_cert_store_stats__doc__, "\n" "Returns quantities of loaded X.509 certificates.\n" "\n" -"X.509 certificates with a CA extension and certificate revocation lists\n" -"inside the context\'s cert store.\n" +"X.509 certificates with a CA extension and certificate revocation\n" +"lists inside the context\'s cert store.\n" "\n" -"NOTE: Certificates in a capath directory aren\'t loaded unless they have\n" -"been used at least once."); +"NOTE: Certificates in a capath directory aren\'t loaded unless they\n" +"have been used at least once."); #define _SSL__SSLCONTEXT_CERT_STORE_STATS_METHODDEF \ {"cert_store_stats", (PyCFunction)_ssl__SSLContext_cert_store_stats, METH_NOARGS, _ssl__SSLContext_cert_store_stats__doc__}, @@ -2305,11 +2308,11 @@ PyDoc_STRVAR(_ssl__SSLContext_get_ca_certs__doc__, "\n" "Returns a list of dicts with information of loaded CA certs.\n" "\n" -"If the optional argument is True, returns a DER-encoded copy of the CA\n" -"certificate.\n" +"If the optional argument is True, returns a DER-encoded copy of the\n" +"CA certificate.\n" "\n" -"NOTE: Certificates in a capath directory aren\'t loaded unless they have\n" -"been used at least once."); +"NOTE: Certificates in a capath directory aren\'t loaded unless they\n" +"have been used at least once."); #define _SSL__SSLCONTEXT_GET_CA_CERTS_METHODDEF \ {"get_ca_certs", _PyCFunction_CAST(_ssl__SSLContext_get_ca_certs), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_get_ca_certs__doc__}, @@ -2970,8 +2973,8 @@ PyDoc_STRVAR(_ssl_RAND_status__doc__, "\n" "Returns True if the OpenSSL PRNG has been seeded with enough data and False if not.\n" "\n" -"It is necessary to seed the PRNG with RAND_add() on some platforms before\n" -"using the ssl() function."); +"It is necessary to seed the PRNG with RAND_add() on some platforms\n" +"before using the ssl() function."); #define _SSL_RAND_STATUS_METHODDEF \ {"RAND_status", (PyCFunction)_ssl_RAND_status, METH_NOARGS, _ssl_RAND_status__doc__}, @@ -3157,11 +3160,11 @@ PyDoc_STRVAR(_ssl_enum_certificates__doc__, "\n" "Retrieve certificates from Windows\' cert store.\n" "\n" -"store_name may be one of \'CA\', \'ROOT\' or \'MY\'. The system may provide\n" -"more cert storages, too. The function returns a list of (bytes,\n" -"encoding_type, trust) tuples. The encoding_type flag can be interpreted\n" -"with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either\n" -"a set of OIDs or the boolean True."); +"store_name may be one of \'CA\', \'ROOT\' or \'MY\'. The system may\n" +"provide more cert storages, too. The function returns a list of\n" +"(bytes, encoding_type, trust) tuples. The encoding_type flag can be\n" +"interpreted with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The\n" +"trust setting is either a set of OIDs or the boolean True."); #define _SSL_ENUM_CERTIFICATES_METHODDEF \ {"enum_certificates", _PyCFunction_CAST(_ssl_enum_certificates), METH_FASTCALL|METH_KEYWORDS, _ssl_enum_certificates__doc__}, @@ -3323,4 +3326,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=e29d5ada294f97bb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aef2e74b706c6106 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 311b64094767110..a0e4c66451101ed 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_testmultiphase_StateAccessType_get_defining_module__doc__, "\n" "Return the module of the defining class.\n" "\n" -"Also tests that result of PyType_GetModuleByDef matches defining_class\'s\n" -"module."); +"Also tests that result of PyType_GetModuleByDef matches\n" +"defining_class\'s module."); #define _TESTMULTIPHASE_STATEACCESSTYPE_GET_DEFINING_MODULE_METHODDEF \ {"get_defining_module", _PyCFunction_CAST(_testmultiphase_StateAccessType_get_defining_module), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testmultiphase_StateAccessType_get_defining_module__doc__}, @@ -165,4 +165,4 @@ _testmultiphase_StateAccessType_get_count(PyObject *self, PyTypeObject *cls, PyO } return _testmultiphase_StateAccessType_get_count_impl((StateAccessTypeObject *)self, cls); } -/*[clinic end generated code: output=8eed2f14292ec986 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aff91f6219a7baca input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_threadmodule.c.h b/Modules/clinic/_threadmodule.c.h index 8455f9929babc19..926bea8e1e419a4 100644 --- a/Modules/clinic/_threadmodule.c.h +++ b/Modules/clinic/_threadmodule.c.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_thread_lock_acquire__doc__, "\n" "Lock the lock.\n" "\n" -"Without argument, this blocks if the lock is already\n" -"locked (even by the same thread), waiting for another thread to release\n" +"Without argument, this blocks if the lock is already locked\n" +"(even by the same thread), waiting for another thread to release\n" "the lock, and return True once the lock is acquired.\n" "With an argument, this will only block if the argument is true,\n" "and the return value reflects whether the lock is acquired.\n" @@ -445,14 +445,14 @@ PyDoc_STRVAR(_thread_RLock_release__doc__, "\n" "Release the lock.\n" "\n" -"Allows another thread that is blocked waiting for\n" -"the lock to acquire the lock. The lock must be in the locked state,\n" +"Allows another thread that is blocked waiting for the lock\n" +"to acquire the lock. The lock must be in the locked state,\n" "and must be locked by the same thread that unlocks it; otherwise a\n" "`RuntimeError` is raised.\n" "\n" -"Do note that if the lock was acquire()d several times in a row by the\n" -"current thread, release() needs to be called as many times for the lock\n" -"to be available for other threads."); +"Do note that if the lock was acquire()d several times in a row by\n" +"the current thread, release() needs to be called as many times for\n" +"the lock to be available for other threads."); #define _THREAD_RLOCK_RELEASE_METHODDEF \ {"release", (PyCFunction)_thread_RLock_release, METH_NOARGS, _thread_RLock_release__doc__}, @@ -740,4 +740,4 @@ _thread_set_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _THREAD_SET_NAME_METHODDEF #define _THREAD_SET_NAME_METHODDEF #endif /* !defined(_THREAD_SET_NAME_METHODDEF) */ -/*[clinic end generated code: output=1255a1520f43f97a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0f1707cbafc0e8f2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 352c2b9e3d410c4..3147cf7f9d2cff3 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -907,7 +907,8 @@ PyDoc_STRVAR(_tkinter_setbusywaitinterval__doc__, "\n" "Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter.\n" "\n" -"It should be set to a divisor of the maximum time between frames in an animation."); +"It should be set to a divisor of the maximum time between frames in\n" +"an animation."); #define _TKINTER_SETBUSYWAITINTERVAL_METHODDEF \ {"setbusywaitinterval", (PyCFunction)_tkinter_setbusywaitinterval, METH_O, _tkinter_setbusywaitinterval__doc__}, @@ -966,4 +967,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=052c067aa69237be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c807adb73e305725 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 00cce91dca43b1c..031a0783aef60bb 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -2331,7 +2331,52 @@ _winapi_ReportEvent(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(_winapi_GetProcessMemoryInfo__doc__, +"GetProcessMemoryInfo($module, handle, /)\n" +"--\n" +"\n" +"Return the memory usage of the given process handle as a dict."); + +#define _WINAPI_GETPROCESSMEMORYINFO_METHODDEF \ + {"GetProcessMemoryInfo", (PyCFunction)_winapi_GetProcessMemoryInfo, METH_O, _winapi_GetProcessMemoryInfo__doc__}, + +static PyObject * +_winapi_GetProcessMemoryInfo_impl(PyObject *module, HANDLE handle); + +static PyObject * +_winapi_GetProcessMemoryInfo(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + HANDLE handle; + + if (!PyArg_Parse(arg, "" F_HANDLE ":GetProcessMemoryInfo", &handle)) { + goto exit; + } + return_value = _winapi_GetProcessMemoryInfo_impl(module, handle); + +exit: + return return_value; +} + +PyDoc_STRVAR(_winapi_GetTickCount64__doc__, +"GetTickCount64($module, /)\n" +"--\n" +"\n" +"Number of milliseconds that have elapsed since the system was started."); + +#define _WINAPI_GETTICKCOUNT64_METHODDEF \ + {"GetTickCount64", (PyCFunction)_winapi_GetTickCount64, METH_NOARGS, _winapi_GetTickCount64__doc__}, + +static PyObject * +_winapi_GetTickCount64_impl(PyObject *module); + +static PyObject * +_winapi_GetTickCount64(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _winapi_GetTickCount64_impl(module); +} + #ifndef _WINAPI_GETSHORTPATHNAME_METHODDEF #define _WINAPI_GETSHORTPATHNAME_METHODDEF #endif /* !defined(_WINAPI_GETSHORTPATHNAME_METHODDEF) */ -/*[clinic end generated code: output=4ab94eaee93a0a90 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=713a8ce97185b017 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index eec47ab2b1f9e1d..1e9f317853d6c2e 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -292,8 +292,8 @@ PyDoc_STRVAR(array_array_buffer_info__doc__, "\n" "Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array\'s contents.\n" "\n" -"The length should be multiplied by the itemsize attribute to calculate\n" -"the buffer length in bytes."); +"The length should be multiplied by the itemsize attribute to\n" +"calculate the buffer length in bytes."); #define ARRAY_ARRAY_BUFFER_INFO_METHODDEF \ {"buffer_info", (PyCFunction)array_array_buffer_info, METH_NOARGS, array_array_buffer_info__doc__}, @@ -335,8 +335,8 @@ PyDoc_STRVAR(array_array_byteswap__doc__, "\n" "Byteswap all items of the array.\n" "\n" -"If the items in the array are not 1, 2, 4, 8 or 16 bytes in size, RuntimeError\n" -"is raised. Note, that for complex types the order of\n" +"If the items in the array are not 1, 2, 4, 8 or 16 bytes in size,\n" +"RuntimeError is raised. Note, that for complex types the order of\n" "components (the real part, followed by imaginary part) is preserved."); #define ARRAY_ARRAY_BYTESWAP_METHODDEF \ @@ -572,9 +572,9 @@ PyDoc_STRVAR(array_array_fromunicode__doc__, "\n" "Extends this array with data from the unicode string ustr.\n" "\n" -"The array must be a unicode type array; otherwise a ValueError is raised.\n" -"Use array.frombytes(ustr.encode(...)) to append Unicode data to an array of\n" -"some other type."); +"The array must be a unicode type array; otherwise a ValueError is\n" +"raised. Use array.frombytes(ustr.encode(...)) to append Unicode\n" +"data to an array of some other type."); #define ARRAY_ARRAY_FROMUNICODE_METHODDEF \ {"fromunicode", (PyCFunction)array_array_fromunicode, METH_O, array_array_fromunicode__doc__}, @@ -605,9 +605,10 @@ PyDoc_STRVAR(array_array_tounicode__doc__, "\n" "Extends this array with data from the unicode string ustr.\n" "\n" -"Convert the array to a unicode string. The array must be a unicode type array;\n" -"otherwise a ValueError is raised. Use array.tobytes().decode() to obtain a\n" -"unicode string from an array of some other type."); +"Convert the array to a unicode string. The array must be a unicode\n" +"type array; otherwise a ValueError is raised. Use\n" +"array.tobytes().decode() to obtain a unicode string from an array of\n" +"some other type."); #define ARRAY_ARRAY_TOUNICODE_METHODDEF \ {"tounicode", (PyCFunction)array_array_tounicode, METH_NOARGS, array_array_tounicode__doc__}, @@ -780,4 +781,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=8699475b51151247 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=32784678e77ac658 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/binascii.c.h b/Modules/clinic/binascii.c.h index ed695758ef998c9..29fa9e87de87c7a 100644 --- a/Modules/clinic/binascii.c.h +++ b/Modules/clinic/binascii.c.h @@ -372,7 +372,8 @@ PyDoc_STRVAR(binascii_a2b_ascii85__doc__, " foldspaces\n" " Allow \'y\' as a short form encoding four spaces.\n" " adobe\n" -" Expect data to be wrapped in \'<~\' and \'~>\' as in Adobe Ascii85.\n" +" Expect data to be terminated with \'~>\' as in Adobe Ascii85, and\n" +" optionally accept leading \'<~\'.\n" " ignorechars\n" " A byte string containing characters to ignore from the input.\n" " canonical\n" @@ -492,7 +493,7 @@ PyDoc_STRVAR(binascii_b2a_ascii85__doc__, " wrapcol\n" " Split result into lines of provided width.\n" " pad\n" -" Pad input to a multiple of 4 before encoding.\n" +" Retain zero-padding bytes at end of output.\n" " adobe\n" " Wrap result in \'<~\' and \'~>\' as in Adobe Ascii85."); @@ -709,7 +710,7 @@ PyDoc_STRVAR(binascii_b2a_base85__doc__, "Base85-code line of data.\n" "\n" " pad\n" -" Pad input to a multiple of 4 before encoding."); +" Retain zero-padding bytes at end of output."); #define BINASCII_B2A_BASE85_METHODDEF \ {"b2a_base85", _PyCFunction_CAST(binascii_b2a_base85), METH_FASTCALL|METH_KEYWORDS, binascii_b2a_base85__doc__}, @@ -1684,4 +1685,4 @@ binascii_b2a_qp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return return_value; } -/*[clinic end generated code: output=b41544f39b0ef681 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=42dd48f323cbb118 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index 7f9e65baf120ea2..ecb5257cf6b2408 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -644,7 +644,8 @@ PyDoc_STRVAR(cmath_log__doc__, "\n" "log(z[, base]) -> the logarithm of z to the given base.\n" "\n" -"If the base is not specified, returns the natural logarithm (base e) of z."); +"If the base is not specified, returns the natural logarithm (base e)\n" +"of z."); #define CMATH_LOG_METHODDEF \ {"log", _PyCFunction_CAST(cmath_log), METH_FASTCALL, cmath_log__doc__}, @@ -882,11 +883,12 @@ PyDoc_STRVAR(cmath_isclose__doc__, "\n" "Return True if a is close in value to b, and False otherwise.\n" "\n" -"For the values to be considered close, the difference between them must be\n" -"smaller than at least one of the tolerances.\n" +"For the values to be considered close, the difference between them must\n" +"be smaller than at least one of the tolerances.\n" "\n" -"-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is\n" -"not close to anything, even itself. inf and -inf are only close to themselves."); +"-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is,\n" +"NaN is not close to anything, even itself. inf and -inf are only close\n" +"to themselves."); #define CMATH_ISCLOSE_METHODDEF \ {"isclose", _PyCFunction_CAST(cmath_isclose), METH_FASTCALL|METH_KEYWORDS, cmath_isclose__doc__}, @@ -985,4 +987,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=631db17fb1c79d66 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7d5ad4cf258526cd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/faulthandler.c.h b/Modules/clinic/faulthandler.c.h index e06cfdcfba29936..07a9dd7dc561e12 100644 --- a/Modules/clinic/faulthandler.c.h +++ b/Modules/clinic/faulthandler.c.h @@ -334,9 +334,9 @@ PyDoc_STRVAR(faulthandler_dump_traceback_later__doc__, "\n" "Dump the traceback of all threads in timeout seconds.\n" "\n" -"If repeat is true, the tracebacks of all threads are dumped every timeout\n" -"seconds. If exit is true, call _exit(1) which is not safe. max_threads\n" -"caps the number of threads dumped."); +"If repeat is true, the tracebacks of all threads are dumped every\n" +"timeout seconds. If exit is true, call _exit(1) which is not safe.\n" +"max_threads caps the number of threads dumped."); #define FAULTHANDLER_DUMP_TRACEBACK_LATER_METHODDEF \ {"dump_traceback_later", _PyCFunction_CAST(faulthandler_dump_traceback_later), METH_FASTCALL|METH_KEYWORDS, faulthandler_dump_traceback_later__doc__}, @@ -782,4 +782,4 @@ faulthandler__raise_exception(PyObject *module, PyObject *const *args, Py_ssize_ #ifndef FAULTHANDLER__RAISE_EXCEPTION_METHODDEF #define FAULTHANDLER__RAISE_EXCEPTION_METHODDEF #endif /* !defined(FAULTHANDLER__RAISE_EXCEPTION_METHODDEF) */ -/*[clinic end generated code: output=2452d767c85130a6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=14815a5f8afe813f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 08275e35413f667..aa743c8f40a565f 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -376,8 +376,8 @@ PyDoc_STRVAR(gc_get_objects__doc__, " generation\n" " Generation to extract the objects from.\n" "\n" -"If generation is not None, return only the objects tracked by the collector\n" -"that are in that generation."); +"If generation is not None, return only the objects tracked by the\n" +"collector that are in that generation."); #define GC_GET_OBJECTS_METHODDEF \ {"get_objects", _PyCFunction_CAST(gc_get_objects), METH_FASTCALL|METH_KEYWORDS, gc_get_objects__doc__}, @@ -520,9 +520,10 @@ PyDoc_STRVAR(gc_freeze__doc__, "\n" "Freeze all current tracked objects and ignore them for future collections.\n" "\n" -"This can be used before a POSIX fork() call to make the gc copy-on-write friendly.\n" -"Note: collection before a POSIX fork() call may free pages for future allocation\n" -"which can cause copy-on-write."); +"This can be used before a POSIX fork() call to make the gc copy-on-write\n" +"friendly.\n" +"Note: collection before a POSIX fork() call may free pages for future\n" +"allocation which can cause copy-on-write."); #define GC_FREEZE_METHODDEF \ {"freeze", (PyCFunction)gc_freeze, METH_NOARGS, gc_freeze__doc__}, @@ -583,4 +584,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=19738854607938db input=a9049054013a1b77]*/ +/*[clinic end generated code: output=756c0e7719b76971 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/hmacmodule.c.h b/Modules/clinic/hmacmodule.c.h index 1ceb2d809e830a2..a31d2ab7f04463c 100644 --- a/Modules/clinic/hmacmodule.c.h +++ b/Modules/clinic/hmacmodule.c.h @@ -187,8 +187,8 @@ PyDoc_STRVAR(_hmac_HMAC_hexdigest__doc__, "\n" "Return hexadecimal digest of the bytes passed to the update() method so far.\n" "\n" -"This may be used to exchange the value safely in email or other non-binary\n" -"environments.\n" +"This may be used to exchange the value safely in email or other\n" +"non-binary environments.\n" "\n" "This method may raise a MemoryError."); @@ -670,4 +670,4 @@ _hmac_compute_blake2b_32(PyObject *module, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=30c0614482d963f5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6ec5948df1c5569a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 49816bfcb42feca..d9917b47dc90376 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -814,8 +814,8 @@ PyDoc_STRVAR(itertools_compress__doc__, "\n" "Return data elements corresponding to true selector elements.\n" "\n" -"Forms a shorter iterator from selected data elements using the selectors to\n" -"choose the data elements."); +"Forms a shorter iterator from selected data elements using the selectors\n" +"to choose the data elements."); static PyObject * itertools_compress_impl(PyTypeObject *type, PyObject *seq1, PyObject *seq2); @@ -980,4 +980,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=7f385837b13edbeb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a34a31f60100e0ff input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index b023299dd9cb2dc..a5e583c180defee 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -198,8 +198,9 @@ PyDoc_STRVAR(math_frexp__doc__, "\n" "Return the mantissa and exponent of x, as pair (m, e).\n" "\n" -"m is a float and e is an int, such that x = m * 2.**e.\n" -"If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0."); +"If x is a finite nonzero number, then m is a float with\n" +"0.5 <= abs(m) < 1.0 and an integer e is such that\n" +"x == m * 2**e exactly. Else, return (x, 0)."); #define MATH_FREXP_METHODDEF \ {"frexp", (PyCFunction)math_frexp, METH_O, math_frexp__doc__}, @@ -1043,8 +1044,8 @@ PyDoc_STRVAR(math_nextafter__doc__, "\n" "If steps is not specified or is None, it defaults to 1.\n" "\n" -"Raises a TypeError, if x or y is not a double, or if steps is not an integer.\n" -"Raises ValueError if steps is negative."); +"Raises a TypeError, if x or y is not a double, or if steps is not\n" +"an integer. Raises ValueError if steps is negative."); #define MATH_NEXTAFTER_METHODDEF \ {"nextafter", _PyCFunction_CAST(math_nextafter), METH_FASTCALL|METH_KEYWORDS, math_nextafter__doc__}, @@ -1163,4 +1164,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=23b2453ba77453e5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3452ce8caa2d1bd7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 7e2480bdace38d2..ba41eab15650e8f 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -529,8 +529,9 @@ PyDoc_STRVAR(_overlapped_Overlapped_getresult__doc__, "\n" "Retrieve result of operation.\n" "\n" -"If wait is true then it blocks until the operation is finished. If wait\n" -"is false and the operation is still pending then an error is raised."); +"If wait is true then it blocks until the operation is finished. If\n" +"wait is false and the operation is still pending then an error is\n" +"raised."); #define _OVERLAPPED_OVERLAPPED_GETRESULT_METHODDEF \ {"getresult", _PyCFunction_CAST(_overlapped_Overlapped_getresult), METH_FASTCALL, _overlapped_Overlapped_getresult__doc__}, @@ -1242,4 +1243,4 @@ _overlapped_Overlapped_WSARecvFromInto(PyObject *self, PyObject *const *args, Py return return_value; } -/*[clinic end generated code: output=3e4cb2b55342cd96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ecaf45a09539599 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ad4c5dd1c9bc084..00a4f6009186af9 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -512,9 +512,9 @@ PyDoc_STRVAR(os_chdir__doc__, "\n" "Change the current working directory to the specified path.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -"If this functionality is unavailable, using it raises an exception."); +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception."); #define OS_CHDIR_METHODDEF \ {"chdir", _PyCFunction_CAST(os_chdir), METH_FASTCALL|METH_KEYWORDS, os_chdir__doc__}, @@ -649,14 +649,15 @@ PyDoc_STRVAR(os_chmod__doc__, "Change the access permissions of a file.\n" "\n" " path\n" -" Path to be modified. May always be specified as a str, bytes, or a path-like object.\n" -" On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" +" Path to be modified. May always be specified as a str, bytes, or\n" +" a path-like object. On some platforms, path may also be specified\n" +" as an open file descriptor. If this functionality is unavailable,\n" +" using it raises an exception.\n" " mode\n" " Operating-system mode bitfield.\n" -" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" -" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" -" Python.\n" +" Be careful when using number literals for *mode*. The conventional\n" +" UNIX notation for numeric modes uses an octal base, which needs to\n" +" be indicated with a ``0o`` prefix in Python.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be relative; path will then be relative to that\n" @@ -765,9 +766,9 @@ PyDoc_STRVAR(os_fchmod__doc__, " The file descriptor of the file to be modified.\n" " mode\n" " Operating-system mode bitfield.\n" -" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" -" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" -" Python.\n" +" Be careful when using number literals for *mode*. The conventional\n" +" UNIX notation for numeric modes uses an octal base, which needs to\n" +" be indicated with a ``0o`` prefix in Python.\n" "\n" "Equivalent to os.chmod(fd, mode)."); @@ -841,8 +842,8 @@ PyDoc_STRVAR(os_lchmod__doc__, "\n" "Change the access permissions of a file, without following symbolic links.\n" "\n" -"If path is a symlink, this affects the link itself rather than the target.\n" -"Equivalent to chmod(path, mode, follow_symlinks=False).\""); +"If path is a symlink, this affects the link itself rather than the\n" +"target. Equivalent to chmod(path, mode, follow_symlinks=False)."); #define OS_LCHMOD_METHODDEF \ {"lchmod", _PyCFunction_CAST(os_lchmod), METH_FASTCALL|METH_KEYWORDS, os_lchmod__doc__}, @@ -916,9 +917,9 @@ PyDoc_STRVAR(os_chflags__doc__, "\n" "Set file flags.\n" "\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, chflags will change flags on the symbolic link itself instead of the\n" -" file the link points to.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, chflags() will change flags on the symbolic link itself\n" +"instead of the file the link points to.\n" "follow_symlinks may not be implemented on your platform. If it is\n" "unavailable, using it will raise a NotImplementedError."); @@ -1329,10 +1330,11 @@ PyDoc_STRVAR(os_chown__doc__, "chown($module, /, path, uid, gid, *, dir_fd=None, follow_symlinks=True)\n" "--\n" "\n" -"Change the owner and group id of path to the numeric uid and gid.\\\n" +"Change the owner and group id of path to the numeric uid and gid.\n" "\n" " path\n" -" Path to be examined; can be string, bytes, a path-like object, or open-file-descriptor int.\n" +" Path to be examined; can be string, bytes, a path-like object, or\n" +" open-file-descriptor int.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be relative; path will then be relative to that\n" @@ -1342,18 +1344,19 @@ PyDoc_STRVAR(os_chown__doc__, " stat will examine the symbolic link itself instead of the file\n" " the link points to.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, chown will modify the symbolic link itself instead of the file the\n" -" link points to.\n" +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, chown will modify the symbolic link itself instead of\n" +"the file the link points to.\n" "It is an error to use dir_fd or follow_symlinks when specifying path as\n" -" an open file descriptor.\n" -"dir_fd and follow_symlinks may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"an open file descriptor.\n" +"dir_fd and follow_symlinks may not be implemented on your platform. If\n" +"they are unavailable, using them will raise a NotImplementedError."); #define OS_CHOWN_METHODDEF \ {"chown", _PyCFunction_CAST(os_chown), METH_FASTCALL|METH_KEYWORDS, os_chown__doc__}, @@ -1641,14 +1644,15 @@ PyDoc_STRVAR(os_link__doc__, "Create a hard link to a file.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "If follow_symlinks is False, and the last element of src is a symbolic\n" -" link, link will create a link to the symbolic link itself instead of the\n" -" file the link points to.\n" -"src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your\n" -" platform. If they are unavailable, using them will raise a\n" -" NotImplementedError."); +"link, link will create a link to the symbolic link itself instead of the\n" +"file the link points to.\n" +"src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on\n" +"your platform. If they are unavailable, using them will raise\n" +"a NotImplementedError."); #define OS_LINK_METHODDEF \ {"link", _PyCFunction_CAST(os_link), METH_FASTCALL|METH_KEYWORDS, os_link__doc__}, @@ -1750,13 +1754,13 @@ PyDoc_STRVAR(os_listdir__doc__, "\n" "Return a list containing the names of the files in the directory.\n" "\n" -"path can be specified as either str, bytes, or a path-like object. If path is bytes,\n" -" the filenames returned will also be bytes; in all other circumstances\n" -" the filenames returned will be str.\n" +"path can be specified as either str, bytes, or a path-like object. If\n" +"path is bytes, the filenames returned will also be bytes; in all other\n" +"circumstances the filenames returned will be str.\n" "If path is None, uses the path=\'.\'.\n" -"On some platforms, path may also be specified as an open file descriptor;\\\n" -" the file descriptor must refer to a directory.\n" -" If this functionality is unavailable, using it raises NotImplementedError.\n" +"On some platforms, path may also be specified as an open file\n" +"descriptor; the file descriptor must refer to a directory. If this\n" +"functionality is unavailable, using it raises NotImplementedError.\n" "\n" "The list is in arbitrary order. It does not include the special\n" "entries \'.\' and \'..\' even if they are present in the directory."); @@ -2709,13 +2713,14 @@ PyDoc_STRVAR(os_mkdir__doc__, "\n" "Create a directory.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError.\n" "\n" -"The mode argument is ignored on Windows. Where it is used, the current umask\n" -"value is first masked out."); +"The mode argument is ignored on Windows. Where it is used, the current\n" +"umask value is first masked out."); #define OS_MKDIR_METHODDEF \ {"mkdir", _PyCFunction_CAST(os_mkdir), METH_FASTCALL|METH_KEYWORDS, os_mkdir__doc__}, @@ -2981,10 +2986,11 @@ PyDoc_STRVAR(os_rename__doc__, "Rename a file or directory.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "src_dir_fd and dst_dir_fd, may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_RENAME_METHODDEF \ {"rename", _PyCFunction_CAST(os_rename), METH_FASTCALL|METH_KEYWORDS, os_rename__doc__}, @@ -3075,10 +3081,11 @@ PyDoc_STRVAR(os_replace__doc__, "Rename a file or directory, overwriting the destination.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "src_dir_fd and dst_dir_fd, may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_REPLACE_METHODDEF \ {"replace", _PyCFunction_CAST(os_replace), METH_FASTCALL|METH_KEYWORDS, os_replace__doc__}, @@ -3168,10 +3175,11 @@ PyDoc_STRVAR(os_rmdir__doc__, "\n" "Remove a directory.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_RMDIR_METHODDEF \ {"rmdir", _PyCFunction_CAST(os_rmdir), METH_FASTCALL|METH_KEYWORDS, os_rmdir__doc__}, @@ -3426,10 +3434,11 @@ PyDoc_STRVAR(os_unlink__doc__, "\n" "Remove a file (same as remove()).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_UNLINK_METHODDEF \ {"unlink", _PyCFunction_CAST(os_unlink), METH_FASTCALL|METH_KEYWORDS, os_unlink__doc__}, @@ -3503,10 +3512,11 @@ PyDoc_STRVAR(os_remove__doc__, "\n" "Remove a file (same as unlink()).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_REMOVE_METHODDEF \ {"remove", _PyCFunction_CAST(os_remove), METH_FASTCALL|METH_KEYWORDS, os_remove__doc__}, @@ -3606,27 +3616,28 @@ PyDoc_STRVAR(os_utime__doc__, "\n" "Set the access and modified time of path.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception.\n" "\n" "If times is not None, it must be a tuple (atime, mtime);\n" -" atime and mtime should be expressed as float seconds since the epoch.\n" +"atime and mtime should be expressed as float seconds since the epoch.\n" "If ns is specified, it must be a tuple (atime_ns, mtime_ns);\n" -" atime_ns and mtime_ns should be expressed as integer nanoseconds\n" -" since the epoch.\n" +"atime_ns and mtime_ns should be expressed as integer nanoseconds\n" +"since the epoch.\n" "If times is None and ns is unspecified, utime uses the current time.\n" "Specifying tuples for both times and ns is an error.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, utime will modify the symbolic link itself instead of the file the\n" -" link points to.\n" -"It is an error to use dir_fd or follow_symlinks when specifying path\n" -" as an open file descriptor.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, utime will modify the symbolic link itself instead of\n" +"the file the link points to.\n" +"It is an error to use dir_fd or follow_symlinks when specifying path as\n" +"an open file descriptor.\n" "dir_fd and follow_symlinks may not be available on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_UTIME_METHODDEF \ {"utime", _PyCFunction_CAST(os_utime), METH_FASTCALL|METH_KEYWORDS, os_utime__doc__}, @@ -3929,7 +3940,8 @@ PyDoc_STRVAR(os_posix_spawn__doc__, " resetids\n" " If the value is `true` the POSIX_SPAWN_RESETIDS will be activated.\n" " setsid\n" -" If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n" +" If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP\n" +" will be activated.\n" " setsigmask\n" " The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" " setsigdef\n" @@ -4082,7 +4094,8 @@ PyDoc_STRVAR(os_posix_spawnp__doc__, " resetids\n" " If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n" " setsid\n" -" If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n" +" If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP\n" +" will be activated.\n" " setsigmask\n" " The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" " setsigdef\n" @@ -4953,8 +4966,8 @@ PyDoc_STRVAR(os_posix_openpt__doc__, "Open and return a file descriptor for a master pseudo-terminal device.\n" "\n" "Performs a posix_openpt() C function call. The oflag argument is used to\n" -"set file status flags and file access modes as specified in the manual page\n" -"of posix_openpt() of your system."); +"set file status flags and file access modes as specified in the manual\n" +"page of posix_openpt() of your system."); #define OS_POSIX_OPENPT_METHODDEF \ {"posix_openpt", (PyCFunction)os_posix_openpt, METH_O, os_posix_openpt__doc__}, @@ -5411,9 +5424,9 @@ PyDoc_STRVAR(os_initgroups__doc__, "\n" "Initialize the group access list.\n" "\n" -"Call the system initgroups() to initialize the group access list with all of\n" -"the groups of which the specified username is a member, plus the specified\n" -"group id."); +"Call the system initgroups() to initialize the group access list with\n" +"all of the groups of which the specified username is a member, plus the\n" +"specified group id."); #define OS_INITGROUPS_METHODDEF \ {"initgroups", _PyCFunction_CAST(os_initgroups), METH_FASTCALL, os_initgroups__doc__}, @@ -5457,9 +5470,9 @@ PyDoc_STRVAR(os_initgroups__doc__, "\n" "Initialize the group access list.\n" "\n" -"Call the system initgroups() to initialize the group access list with all of\n" -"the groups of which the specified username is a member, plus the specified\n" -"group id."); +"Call the system initgroups() to initialize the group access list with\n" +"all of the groups of which the specified username is a member, plus the\n" +"specified group id."); #define OS_INITGROUPS_METHODDEF \ {"initgroups", _PyCFunction_CAST(os_initgroups), METH_FASTCALL, os_initgroups__doc__}, @@ -5612,7 +5625,8 @@ PyDoc_STRVAR(os_getppid__doc__, "Return the parent\'s process id.\n" "\n" "If the parent process has already exited, Windows machines will still\n" -"return its id; others systems will return the id of the \'init\' process (1)."); +"return its id; others systems will return the id of the \'init\' proces\n" +"(1)."); #define OS_GETPPID_METHODDEF \ {"getppid", (PyCFunction)os_getppid, METH_NOARGS, os_getppid__doc__}, @@ -6162,8 +6176,8 @@ PyDoc_STRVAR(os_waitid__doc__, " Constructed from the ORing of one or more of WEXITED, WSTOPPED\n" " or WCONTINUED and additionally may be ORed with WNOHANG or WNOWAIT.\n" "\n" -"Returns either waitid_result or None if WNOHANG is specified and there are\n" -"no children in a waitable state."); +"Returns either waitid_result or None if WNOHANG is specified and there\n" +"are no children in a waitable state."); #define OS_WAITID_METHODDEF \ {"waitid", _PyCFunction_CAST(os_waitid), METH_FASTCALL, os_waitid__doc__}, @@ -6324,8 +6338,8 @@ PyDoc_STRVAR(os_pidfd_open__doc__, "\n" "Return a file descriptor referring to the process *pid*.\n" "\n" -"The descriptor can be used to perform process management without races and\n" -"signals."); +"The descriptor can be used to perform process management without races\n" +"and signals."); #define OS_PIDFD_OPEN_METHODDEF \ {"pidfd_open", _PyCFunction_CAST(os_pidfd_open), METH_FASTCALL|METH_KEYWORDS, os_pidfd_open__doc__}, @@ -6393,6 +6407,93 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec #endif /* (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ +#if (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) + +PyDoc_STRVAR(os_pidfd_getfd__doc__, +"pidfd_getfd($module, /, pidfd, targetfd, *, flags=0)\n" +"--\n" +"\n" +"Duplicate a file descriptor from the process referred to by *pidfd*.\n" +"\n" +" pidfd\n" +" A process file descriptor.\n" +" targetfd\n" +" The file descriptor to duplicate from the target process.\n" +" flags\n" +" Reserved, must be 0."); + +#define OS_PIDFD_GETFD_METHODDEF \ + {"pidfd_getfd", _PyCFunction_CAST(os_pidfd_getfd), METH_FASTCALL|METH_KEYWORDS, os_pidfd_getfd__doc__}, + +static PyObject * +os_pidfd_getfd_impl(PyObject *module, int pidfd, int targetfd, + unsigned int flags); + +static PyObject * +os_pidfd_getfd(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(pidfd), &_Py_ID(targetfd), &_Py_ID(flags), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"pidfd", "targetfd", "flags", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "pidfd_getfd", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + int pidfd; + int targetfd; + unsigned int flags = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 2, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + pidfd = PyLong_AsInt(args[0]); + if (pidfd == -1 && PyErr_Occurred()) { + goto exit; + } + targetfd = PyLong_AsInt(args[1]); + if (targetfd == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (!_PyLong_UnsignedInt_Converter(args[2], &flags)) { + goto exit; + } +skip_optional_kwonly: + return_value = os_pidfd_getfd_impl(module, pidfd, targetfd, flags); + +exit: + return return_value; +} + +#endif /* (defined(__linux__) && defined(__NR_pidfd_getfd) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ + #if defined(HAVE_SETNS) PyDoc_STRVAR(os_setns__doc__, @@ -6549,8 +6650,9 @@ PyDoc_STRVAR(os_readlink__doc__, "\n" "Return a string representing the path to which the symbolic link points.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -"and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" "\n" "dir_fd may not be implemented on your platform. If it is unavailable,\n" "using it will raise a NotImplementedError."); @@ -6632,14 +6734,15 @@ PyDoc_STRVAR(os_symlink__doc__, "Create a symbolic link pointing to src named dst.\n" "\n" "target_is_directory is required on Windows if the target is to be\n" -" interpreted as a directory. (On Windows, symlink requires\n" -" Windows 6.0 or greater, and raises a NotImplementedError otherwise.)\n" -" target_is_directory is ignored on non-Windows platforms.\n" +"interpreted as a directory. (On Windows, symlink requires Windows 6.0\n" +"or greater, and raises a NotImplementedError otherwise.)\n" +"target_is_directory is ignored on non-Windows platforms.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_SYMLINK_METHODDEF \ {"symlink", _PyCFunction_CAST(os_symlink), METH_FASTCALL|METH_KEYWORDS, os_symlink__doc__}, @@ -7307,10 +7410,11 @@ PyDoc_STRVAR(os_open__doc__, "\n" "Open a file for low level IO. Returns a file descriptor (integer).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_OPEN_METHODDEF \ {"open", _PyCFunction_CAST(os_open), METH_FASTCALL|METH_KEYWORDS, os_open__doc__}, @@ -7684,7 +7788,8 @@ PyDoc_STRVAR(os_lseek__doc__, " - SEEK_CUR: seek from the current file position.\n" " - SEEK_END: seek from the end of the file.\n" "\n" -"The return value is the number of bytes relative to the beginning of the file."); +"The return value is the number of bytes relative to the beginning of\n" +"the file."); #define OS_LSEEK_METHODDEF \ {"lseek", _PyCFunction_CAST(os_lseek), METH_FASTCALL, os_lseek__doc__}, @@ -7775,15 +7880,15 @@ PyDoc_STRVAR(os_readinto__doc__, "\n" "Read into a buffer object from a file descriptor.\n" "\n" -"The buffer should be mutable and bytes-like. On success, returns the number of\n" -"bytes read. Less bytes may be read than the size of the buffer. The underlying\n" -"system call will be retried when interrupted by a signal, unless the signal\n" -"handler raises an exception. Other errors will not be retried and an error will\n" -"be raised.\n" +"The buffer should be mutable and bytes-like. On success, returns the\n" +"number of bytes read. Less bytes may be read than the size of the\n" +"buffer. The underlying system call will be retried when interrupted by\n" +"a signal, unless the signal handler raises an exception. Other errors\n" +"will not be retried and an error will be raised.\n" "\n" -"Returns 0 if *fd* is at end of file or if the provided *buffer* has length 0\n" -"(which can be used to check for errors without reading data). Never returns\n" -"negative."); +"Returns 0 if *fd* is at end of file or if the provided *buffer* has\n" +"length 0 (which can be used to check for errors without reading data).\n" +"Never returns negative."); #define OS_READINTO_METHODDEF \ {"readinto", _PyCFunction_CAST(os_readinto), METH_FASTCALL, os_readinto__doc__}, @@ -7938,14 +8043,15 @@ PyDoc_STRVAR(os_preadv__doc__, "\n" "Reads from a file descriptor into a number of mutable bytes-like objects.\n" "\n" -"Combines the functionality of readv() and pread(). As readv(), it will\n" -"transfer data into each buffer until it is full and then move on to the next\n" -"buffer in the sequence to hold the rest of the data. Its fourth argument,\n" -"specifies the file offset at which the input operation is to be performed. It\n" -"will return the total number of bytes read (which can be less than the total\n" -"capacity of all the objects).\n" +"Combines the functionality of readv() and pread(). As readv(), it will\n" +"transfer data into each buffer until it is full and then move on to the\n" +"next buffer in the sequence to hold the rest of the data. Its fourth\n" +"argument, specifies the file offset at which the input operation is to\n" +"be performed. It will return the total number of bytes read (which can\n" +"be less than the total capacity of all the objects).\n" "\n" -"The flags argument contains a bitwise OR of zero or more of the following flags:\n" +"The flags argument contains a bitwise OR of zero or more of the\n" +"following flags:\n" "\n" "- RWF_HIPRI\n" "- RWF_NOWAIT\n" @@ -8679,14 +8785,16 @@ PyDoc_STRVAR(os_pwritev__doc__, "\n" "Writes the contents of bytes-like objects to a file descriptor at a given offset.\n" "\n" -"Combines the functionality of writev() and pwrite(). All buffers must be a sequence\n" -"of bytes-like objects. Buffers are processed in array order. Entire contents of first\n" -"buffer is written before proceeding to second, and so on. The operating system may\n" -"set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used.\n" -"This function writes the contents of each object to the file descriptor and returns\n" -"the total number of bytes written.\n" +"Combines the functionality of writev() and pwrite(). All buffers must be\n" +"a sequence of bytes-like objects. Buffers are processed in array order.\n" +"Entire contents of first buffer is written before proceeding to second,\n" +"and so on. The operating system may set a limit (sysconf() value\n" +"SC_IOV_MAX) on the number of buffers that can be used.\n" +"This function writes the contents of each object to the file descriptor\n" +"and returns the total number of bytes written.\n" "\n" -"The flags argument contains a bitwise OR of zero or more of the following flags:\n" +"The flags argument contains a bitwise OR of zero or more of the\n" +"following flags:\n" "\n" "- RWF_DSYNC\n" "- RWF_SYNC\n" @@ -9001,10 +9109,11 @@ PyDoc_STRVAR(os_mkfifo__doc__, "\n" "Create a \"fifo\" (a POSIX named pipe).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_MKFIFO_METHODDEF \ {"mkfifo", _PyCFunction_CAST(os_mkfifo), METH_FASTCALL|METH_KEYWORDS, os_mkfifo__doc__}, @@ -9096,17 +9205,18 @@ PyDoc_STRVAR(os_mknod__doc__, "\n" "Create a node in the file system.\n" "\n" -"Create a node in the file system (file, device special file or named pipe)\n" -"at path. mode specifies both the permissions to use and the\n" +"Create a node in the file system (file, device special file or named\n" +"pipe) at path. mode specifies both the permissions to use and the\n" "type of node to be created, being combined (bitwise OR) with one of\n" -"S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set on mode,\n" -"device defines the newly created device special file (probably using\n" -"os.makedev()). Otherwise device is ignored.\n" +"S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set\n" +"on mode, device defines the newly created device special file (probably\n" +"using os.makedev()). Otherwise device is ignored.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_MKNOD_METHODDEF \ {"mknod", _PyCFunction_CAST(os_mknod), METH_FASTCALL|METH_KEYWORDS, os_mknod__doc__}, @@ -9352,8 +9462,9 @@ PyDoc_STRVAR(os_truncate__doc__, "\n" "Truncate a file, specified by path, to a specific length.\n" "\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_TRUNCATE_METHODDEF \ {"truncate", _PyCFunction_CAST(os_truncate), METH_FASTCALL|METH_KEYWORDS, os_truncate__doc__}, @@ -9427,7 +9538,8 @@ PyDoc_STRVAR(os_posix_fallocate__doc__, "Ensure a file has allocated at least a particular number of bytes on disk.\n" "\n" "Ensure that the file specified by fd encompasses a range of bytes\n" -"starting at offset bytes from the beginning and continuing for length bytes."); +"starting at offset bytes from the beginning and continuing for length\n" +"bytes."); #define OS_POSIX_FALLOCATE_METHODDEF \ {"posix_fallocate", _PyCFunction_CAST(os_posix_fallocate), METH_FASTCALL, os_posix_fallocate__doc__}, @@ -9473,8 +9585,8 @@ PyDoc_STRVAR(os_posix_fadvise__doc__, "\n" "Announce an intention to access data in a specific pattern.\n" "\n" -"Announce an intention to access data in a specific pattern, thus allowing\n" -"the kernel to make optimizations.\n" +"Announce an intention to access data in a specific pattern, thus\n" +"allowing the kernel to make optimizations.\n" "The advice applies to the region of the file specified by fd starting at\n" "offset and continuing for length bytes.\n" "advice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL,\n" @@ -10296,8 +10408,9 @@ PyDoc_STRVAR(os_statvfs__doc__, "Perform a statvfs system call on the given path.\n" "\n" "path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_STATVFS_METHODDEF \ {"statvfs", _PyCFunction_CAST(os_statvfs), METH_FASTCALL|METH_KEYWORDS, os_statvfs__doc__}, @@ -10480,8 +10593,9 @@ PyDoc_STRVAR(os_pathconf__doc__, "Return the configuration limit name for the file or directory path.\n" "\n" "If there is no limit, return -1.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_PATHCONF_METHODDEF \ {"pathconf", _PyCFunction_CAST(os_pathconf), METH_FASTCALL|METH_KEYWORDS, os_pathconf__doc__}, @@ -10624,8 +10738,8 @@ PyDoc_STRVAR(os_abort__doc__, "\n" "Abort the interpreter immediately.\n" "\n" -"This function \'dumps core\' or otherwise fails in the hardest way possible\n" -"on the hosting operating system. This function never returns."); +"This function \'dumps core\' or otherwise fails in the hardest way\n" +"possible on the hosting operating system. This function never returns."); #define OS_ABORT_METHODDEF \ {"abort", (PyCFunction)os_abort, METH_NOARGS, os_abort__doc__}, @@ -11013,10 +11127,11 @@ PyDoc_STRVAR(os_getxattr__doc__, "\n" "Return the value of extended attribute attribute on path.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, getxattr will examine the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, getxattr will examine the symbolic link itself\n" +"instead of the file the link points to."); #define OS_GETXATTR_METHODDEF \ {"getxattr", _PyCFunction_CAST(os_getxattr), METH_FASTCALL|METH_KEYWORDS, os_getxattr__doc__}, @@ -11103,10 +11218,11 @@ PyDoc_STRVAR(os_setxattr__doc__, "\n" "Set extended attribute attribute on path to value.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, setxattr will modify the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, setxattr will modify the symbolic link itself instead\n" +"of the file the link points to."); #define OS_SETXATTR_METHODDEF \ {"setxattr", _PyCFunction_CAST(os_setxattr), METH_FASTCALL|METH_KEYWORDS, os_setxattr__doc__}, @@ -11214,10 +11330,11 @@ PyDoc_STRVAR(os_removexattr__doc__, "\n" "Remove extended attribute attribute on path.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, removexattr will modify the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, removexattr will modify the symbolic link itself\n" +"instead of the file the link points to."); #define OS_REMOVEXATTR_METHODDEF \ {"removexattr", _PyCFunction_CAST(os_removexattr), METH_FASTCALL|METH_KEYWORDS, os_removexattr__doc__}, @@ -11303,11 +11420,12 @@ PyDoc_STRVAR(os_listxattr__doc__, "\n" "Return a list of extended attributes on path.\n" "\n" -"path may be either None, a string, a path-like object, or an open file descriptor.\n" -"if path is None, listxattr will examine the current directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, listxattr will examine the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either None, a string, a path-like object, or an open file\n" +"descriptor. If path is None, listxattr will examine the current\n" +"directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, listxattr will examine the symbolic link itself instead\n" +"of the file the link points to."); #define OS_LISTXATTR_METHODDEF \ {"listxattr", _PyCFunction_CAST(os_listxattr), METH_FASTCALL|METH_KEYWORDS, os_listxattr__doc__}, @@ -12329,9 +12447,9 @@ PyDoc_STRVAR(os_scandir__doc__, "\n" "Return an iterator of DirEntry objects for given path.\n" "\n" -"path can be specified as either str, bytes, or a path-like object. If path\n" -"is bytes, the names of yielded DirEntry objects will also be bytes; in\n" -"all other circumstances they will be str.\n" +"path can be specified as either str, bytes, or a path-like object. If\n" +"path is bytes, the names of yielded DirEntry objects will also be bytes;\n" +"in all other circumstances they will be str.\n" "\n" "If path is None, uses the path=\'.\'."); @@ -12403,9 +12521,9 @@ PyDoc_STRVAR(os_fspath__doc__, "\n" "Return the file system path representation of the object.\n" "\n" -"If the object is str or bytes, then allow it to pass through as-is. If the\n" -"object defines __fspath__(), then return the result of that method. All other\n" -"types raise a TypeError."); +"If the object is str or bytes, then allow it to pass through as-is. If\n" +"the object defines __fspath__(), then return the result of that method.\n" +"All other types raise a TypeError."); #define OS_FSPATH_METHODDEF \ {"fspath", _PyCFunction_CAST(os_fspath), METH_FASTCALL|METH_KEYWORDS, os_fspath__doc__}, @@ -12699,8 +12817,8 @@ PyDoc_STRVAR(os_waitstatus_to_exitcode__doc__, "On Windows, return status shifted right by 8 bits.\n" "\n" "On Unix, if the process is being traced or if waitpid() was called with\n" -"WUNTRACED option, the caller must first check if WIFSTOPPED(status) is true.\n" -"This function must not be called if WIFSTOPPED(status) is true."); +"WUNTRACED option, the caller must first check if WIFSTOPPED(status) is\n" +"true. This function must not be called if WIFSTOPPED(status) is true."); #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF \ {"waitstatus_to_exitcode", _PyCFunction_CAST(os_waitstatus_to_exitcode), METH_FASTCALL|METH_KEYWORDS, os_waitstatus_to_exitcode__doc__}, @@ -13296,6 +13414,10 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #define OS_PIDFD_OPEN_METHODDEF #endif /* !defined(OS_PIDFD_OPEN_METHODDEF) */ +#ifndef OS_PIDFD_GETFD_METHODDEF + #define OS_PIDFD_GETFD_METHODDEF +#endif /* !defined(OS_PIDFD_GETFD_METHODDEF) */ + #ifndef OS_SETNS_METHODDEF #define OS_SETNS_METHODDEF #endif /* !defined(OS_SETNS_METHODDEF) */ @@ -13611,4 +13733,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=e709b8b783fbc261 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=250ea2e34fdd133f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index ff2e28269dc927e..1a07726d303ecad 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -218,8 +218,9 @@ PyDoc_STRVAR(pyexpat_xmlparser_GetInputContext__doc__, "\n" "Return the untranslated text of the input that caused the current event.\n" "\n" -"If the event was generated by a large amount of text (such as a start tag\n" -"for an element with many attributes), not all of the text may be available."); +"If the event was generated by a large amount of text (such as\n" +"a start tag for an element with many attributes), not all of the\n" +"text may be available."); #define PYEXPAT_XMLPARSER_GETINPUTCONTEXT_METHODDEF \ {"GetInputContext", (PyCFunction)pyexpat_xmlparser_GetInputContext, METH_NOARGS, pyexpat_xmlparser_GetInputContext__doc__}, @@ -357,9 +358,10 @@ PyDoc_STRVAR(pyexpat_xmlparser_UseForeignDTD__doc__, "\n" "Allows the application to provide an artificial external subset if one is not specified as part of the document instance.\n" "\n" -"This readily allows the use of a \'default\' document type controlled by the\n" -"application, while still getting the advantage of providing document type\n" -"information to the parser. \'flag\' defaults to True if not provided."); +"This readily allows the use of a \'default\' document type controlled\n" +"by the application, while still getting the advantage of providing\n" +"document type information to the parser. \'flag\' defaults to True if\n" +"not provided."); #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF \ {"UseForeignDTD", _PyCFunction_CAST(pyexpat_xmlparser_UseForeignDTD), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_UseForeignDTD__doc__}, @@ -417,14 +419,15 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThresho "\n" "Sets the number of output bytes needed to activate protection against billion laughs attacks.\n" "\n" -"The number of output bytes includes amplification from entity expansion\n" -"and reading DTD files.\n" +"The number of output bytes includes amplification from entity\n" +"expansion and reading DTD files.\n" "\n" -"Parser objects usually have a protection activation threshold of 8 MiB,\n" -"but the actual default value depends on the underlying Expat library.\n" +"Parser objects usually have a protection activation threshold of\n" +"8 MiB, but the actual default value depends on the underlying Expat\n" +"library.\n" "\n" -"Activation thresholds below 4 MiB are known to break support for DITA 1.3\n" -"payload and are hence not recommended."); +"Activation thresholds below 4 MiB are known to break support for\n" +"DITA 1.3 payload and are hence not recommended."); #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF \ {"SetBillionLaughsAttackProtectionActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold__doc__}, @@ -479,18 +482,21 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplificat "\n" "Sets the maximum tolerated amplification factor for protection against billion laughs attacks.\n" "\n" -"The amplification factor is calculated as \"(direct + indirect) / direct\"\n" -"while parsing, where \"direct\" is the number of bytes read from the primary\n" -"document in parsing and \"indirect\" is the number of bytes added by expanding\n" -"entities and reading external DTD files, combined.\n" +"The amplification factor is calculated as \"(direct + indirect) /\n" +"direct\" while parsing, where \"direct\" is the number of bytes read\n" +"from the primary document in parsing and \"indirect\" is the number of\n" +"bytes added by expanding entities and reading external DTD files,\n" +"combined.\n" "\n" -"The \'max_factor\' value must be a non-NaN floating point value greater than\n" -"or equal to 1.0. Amplification factors greater than 30,000 can be observed\n" -"in the middle of parsing even with benign files in practice. In particular,\n" -"the activation threshold should be carefully chosen to avoid false positives.\n" +"The \'max_factor\' value must be a non-NaN floating point value\n" +"greater than or equal to 1.0. Amplification factors greater than\n" +"30,000 can be observed in the middle of parsing even with benign\n" +"files in practice. In particular, the activation threshold should\n" +"be carefully chosen to avoid false positives.\n" "\n" "Parser objects usually have a maximum amplification factor of 100,\n" -"but the actual default value depends on the underlying Expat library."); +"but the actual default value depends on the underlying Expat\n" +"library."); #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF \ {"SetBillionLaughsAttackProtectionMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification__doc__}, @@ -551,8 +557,9 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__, "\n" "Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM.\n" "\n" -"Parser objects usually have an allocation activation threshold of 64 MiB,\n" -"but the actual default value depends on the underlying Expat library."); +"Parser objects usually have an allocation activation threshold of\n" +"64 MiB, but the actual default value depends on the underlying Expat\n" +"library."); #define PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF \ {"SetAllocTrackerActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__}, @@ -606,18 +613,20 @@ PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__, "\n" "Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated.\n" "\n" -"The amplification factor is calculated as \"allocated / direct\" while parsing,\n" -"where \"direct\" is the number of bytes read from the primary document in parsing\n" -"and \"allocated\" is the number of bytes of dynamic memory allocated in the parser\n" -"hierarchy.\n" +"The amplification factor is calculated as \"allocated / direct\" while\n" +"parsing, where \"direct\" is the number of bytes read from the primary\n" +"document in parsing and \"allocated\" is the number of bytes of\n" +"dynamic memory allocated in the parser hierarchy.\n" "\n" -"The \'max_factor\' value must be a non-NaN floating point value greater than\n" -"or equal to 1.0. Amplification factors greater than 100.0 can be observed\n" -"near the start of parsing even with benign files in practice. In particular,\n" -"the activation threshold should be carefully chosen to avoid false positives.\n" +"The \'max_factor\' value must be a non-NaN floating point value\n" +"greater than or equal to 1.0. Amplification factors greater than\n" +"100.0 can be observed near the start of parsing even with benign\n" +"files in practice. In particular, the activation threshold should\n" +"be carefully chosen to avoid false positives.\n" "\n" "Parser objects usually have a maximum amplification factor of 100,\n" -"but the actual default value depends on the underlying Expat library."); +"but the actual default value depends on the underlying Expat\n" +"library."); #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF \ {"SetAllocTrackerMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__}, @@ -830,4 +839,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF) */ -/*[clinic end generated code: output=81101a16a409daf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=270a0bfe3300e8a1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 3952054e9e32bfb..c1c8ad40e724f53 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -16,7 +16,8 @@ PyDoc_STRVAR(select_select__doc__, "\n" "Wait until one or more file descriptors are ready for some kind of I/O.\n" "\n" -"The first three arguments are iterables of file descriptors to be waited for:\n" +"The first three arguments are iterables of file descriptors to be waited\n" +"for:\n" "rlist -- wait until ready for reading\n" "wlist -- wait until ready for writing\n" "xlist -- wait for an \"exceptional condition\"\n" @@ -29,9 +30,9 @@ PyDoc_STRVAR(select_select__doc__, "a non-integer to specify fractions of seconds. If it is absent\n" "or None, the call will never time out.\n" "\n" -"The return value is a tuple of three lists corresponding to the first three\n" -"arguments; each contains the subset of the corresponding file descriptors\n" -"that are ready.\n" +"The return value is a tuple of three lists corresponding to the first\n" +"three arguments; each contains the subset of the corresponding file\n" +"descriptors that are ready.\n" "\n" "*** IMPORTANT NOTICE ***\n" "On Windows, only sockets are supported; on Unix, all file\n" @@ -214,8 +215,8 @@ PyDoc_STRVAR(select_poll_poll__doc__, " The maximum time to wait in milliseconds, or else None (or a negative\n" " value) to wait indefinitely.\n" "\n" -"Returns a list containing any descriptors that have events or errors to\n" -"report, as a list of (fd, event) 2-tuples."); +"Returns a list containing any descriptors that have events or errors\n" +"to report, as a list of (fd, event) 2-tuples."); #define SELECT_POLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_poll_poll), METH_FASTCALL, select_poll_poll__doc__}, @@ -396,11 +397,11 @@ PyDoc_STRVAR(select_devpoll_poll__doc__, "Polls the set of registered file descriptors.\n" "\n" " timeout\n" -" The maximum time to wait in milliseconds, or else None (or a negative\n" -" value) to wait indefinitely.\n" +" The maximum time to wait in milliseconds, or else None (or\n" +" a negative value) to wait indefinitely.\n" "\n" -"Returns a list containing any descriptors that have events or errors to\n" -"report, as a list of (fd, event) 2-tuples."); +"Returns a list containing any descriptors that have events or errors\n" +"to report, as a list of (fd, event) 2-tuples."); #define SELECT_DEVPOLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_devpoll_poll), METH_FASTCALL, select_devpoll_poll__doc__}, @@ -498,8 +499,8 @@ PyDoc_STRVAR(select_poll__doc__, "\n" "Returns a polling object.\n" "\n" -"This object supports registering and unregistering file descriptors, and then\n" -"polling them for I/O events."); +"This object supports registering and unregistering file descriptors, and\n" +"then polling them for I/O events."); #define SELECT_POLL_METHODDEF \ {"poll", (PyCFunction)select_poll, METH_NOARGS, select_poll__doc__}, @@ -523,8 +524,8 @@ PyDoc_STRVAR(select_devpoll__doc__, "\n" "Returns a polling object.\n" "\n" -"This object supports registering and unregistering file descriptors, and then\n" -"polling them for I/O events."); +"This object supports registering and unregistering file descriptors, and\n" +"then polling them for I/O events."); #define SELECT_DEVPOLL_METHODDEF \ {"devpoll", (PyCFunction)select_devpoll, METH_NOARGS, select_devpoll__doc__}, @@ -978,8 +979,8 @@ PyDoc_STRVAR(select_epoll_poll__doc__, " maxevents\n" " the maximum number of events returned; -1 means no limit\n" "\n" -"Returns a list containing any descriptors that have events to report,\n" -"as a list of (fd, events) 2-tuples."); +"Returns a list containing any descriptors that have events to\n" +"report, as a list of (fd, events) 2-tuples."); #define SELECT_EPOLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_epoll_poll), METH_FASTCALL|METH_KEYWORDS, select_epoll_poll__doc__}, @@ -1399,4 +1400,4 @@ select_kqueue_control(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=52e3be5cc66cf1b6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a1ac666294fd14bd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 9fd24d15bf25004..ca47033446074cd 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -138,11 +138,12 @@ PyDoc_STRVAR(signal_signal__doc__, "Set the action for the given signal.\n" "\n" "The action can be SIG_DFL, SIG_IGN, or a callable Python object.\n" -"The previous action is returned. See getsignal() for possible return values.\n" +"The previous action is returned. See getsignal() for possible return\n" +"values.\n" "\n" "*** IMPORTANT NOTICE ***\n" -"A signal handler function is called with two arguments:\n" -"the first is the signal number, the second is the interrupted stack frame."); +"A signal handler function is called with two arguments: the first is\n" +"the signal number, the second is the interrupted stack frame."); #define SIGNAL_SIGNAL_METHODDEF \ {"signal", _PyCFunction_CAST(signal_signal), METH_FASTCALL, signal_signal__doc__}, @@ -362,8 +363,8 @@ PyDoc_STRVAR(signal_setitimer__doc__, "\n" "Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF).\n" "\n" -"The timer will fire after value seconds and after that every interval seconds.\n" -"The itimer can be cleared by setting seconds to zero.\n" +"The timer will fire after value seconds and after that every interval\n" +"seconds. The itimer can be cleared by setting seconds to zero.\n" "\n" "Returns old values as a tuple: (delay, interval)."); @@ -508,8 +509,8 @@ PyDoc_STRVAR(signal_sigwait__doc__, "Wait for a signal.\n" "\n" "Suspend execution of the calling thread until the delivery of one of the\n" -"signals specified in the signal set sigset. The function accepts the signal\n" -"and returns the signal number."); +"signals specified in the signal set sigset. The function accepts the\n" +"signal and returns the signal number."); #define SIGNAL_SIGWAIT_METHODDEF \ {"sigwait", (PyCFunction)signal_sigwait, METH_O, signal_sigwait__doc__}, @@ -794,4 +795,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=42e20d118435d7fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0731d6f05c42c09a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index e0cc1c50dcbac30..b565e7516d50f33 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -36,7 +36,8 @@ PyDoc_STRVAR(_socket_socket_send__doc__, "Send a data string to the socket.\n" "\n" "For the optional flags argument, see the Unix manual.\n" -"Return the number of bytes sent; this may be less than len(data) if the network is busy."); +"Return the number of bytes sent; this may be less than len(data) if\n" +"the network is busy."); #define _SOCKET_SOCKET_SEND_METHODDEF \ {"send", _PyCFunction_CAST(_socket_socket_send), METH_FASTCALL, _socket_socket_send__doc__}, @@ -84,7 +85,8 @@ PyDoc_STRVAR(_socket_socket_sendall__doc__, "\n" "For the optional flags argument, see the Unix manual.\n" "This calls send() repeatedly until all data is sent.\n" -"If an error occurs, it\'s impossible to tell how much data has been sent."); +"If an error occurs, it\'s impossible to tell how much data has been\n" +"sent."); #define _SOCKET_SOCKET_SENDALL_METHODDEF \ {"sendall", _PyCFunction_CAST(_socket_socket_sendall), METH_FASTCALL, _socket_socket_sendall__doc__}, @@ -140,13 +142,13 @@ PyDoc_STRVAR(_socket_socket_sendmsg__doc__, "data as an iterable of bytes-like objects (e.g. bytes objects).\n" "The ancdata argument specifies the ancillary data (control messages)\n" "as an iterable of zero or more tuples (cmsg_level, cmsg_type,\n" -"cmsg_data), where cmsg_level and cmsg_type are integers specifying the\n" -"protocol level and protocol-specific type respectively, and cmsg_data\n" -"is a bytes-like object holding the associated data. The flags\n" -"argument defaults to 0 and has the same meaning as for send(). If\n" -"address is supplied and not None, it sets a destination address for\n" -"the message. The return value is the number of bytes of non-ancillary\n" -"data sent."); +"cmsg_data), where cmsg_level and cmsg_type are integers specifying\n" +"the protocol level and protocol-specific type respectively, and\n" +"cmsg_data is a bytes-like object holding the associated data. The\n" +"flags argument defaults to 0 and has the same meaning as for send().\n" +"If address is supplied and not None, it sets a destination address\n" +"for the message. The return value is the number of bytes of\n" +"non-ancillary data sent."); #define _SOCKET_SOCKET_SENDMSG_METHODDEF \ {"sendmsg", _PyCFunction_CAST(_socket_socket_sendmsg), METH_FASTCALL, _socket_socket_sendmsg__doc__}, @@ -541,4 +543,4 @@ _socket_if_indextoname(PyObject *module, PyObject *arg) #ifndef _SOCKET_IF_INDEXTONAME_METHODDEF #define _SOCKET_IF_INDEXTONAME_METHODDEF #endif /* !defined(_SOCKET_IF_INDEXTONAME_METHODDEF) */ -/*[clinic end generated code: output=36051ebf6ad1e6f8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0b1fa78ac6589353 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 83f5a4f6e9f8820..35522bef1dcae9d 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -270,7 +270,8 @@ PyDoc_STRVAR(termios_tcsetwinsize__doc__, "Set the tty winsize for file descriptor fd.\n" "\n" "The winsize to be set is taken from the winsize argument, which\n" -"is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize()."); +"is a two-item tuple (ws_row, ws_col) like the one returned by\n" +"tcgetwinsize()."); #define TERMIOS_TCSETWINSIZE_METHODDEF \ {"tcsetwinsize", (PyCFunction)(void(*)(void))termios_tcsetwinsize, METH_FASTCALL, termios_tcsetwinsize__doc__}, @@ -299,4 +300,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c6c6192583b0da36 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d2176c4d9043d3cc input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index f8fd111754a894a..620e483d5a759a7 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -503,8 +503,8 @@ PyDoc_STRVAR(zlib_Decompress_decompress__doc__, " Unconsumed input data will be stored in\n" " the unconsumed_tail attribute.\n" "\n" -"After calling this function, some of the input data may still be stored in\n" -"internal buffers for later processing.\n" +"After calling this function, some of the input data may still be\n" +"stored in internal buffers for later processing.\n" "Call the flush() method to clear these buffers."); #define ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF \ @@ -914,18 +914,19 @@ PyDoc_STRVAR(zlib__ZlibDecompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define ZLIB__ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(zlib__ZlibDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, zlib__ZlibDecompressor_decompress__doc__}, @@ -1402,4 +1403,4 @@ zlib_crc32_combine(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=13627e14206d3552 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c9a60fe6600a2e4d input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 1e9f9ae051a0b12..7c736f4610bb988 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -948,12 +948,13 @@ cmath.log log(z[, base]) -> the logarithm of z to the given base. -If the base is not specified, returns the natural logarithm (base e) of z. +If the base is not specified, returns the natural logarithm (base e) +of z. [clinic start generated code]*/ static PyObject * cmath_log_impl(PyObject *module, Py_complex x, PyObject *y_obj) -/*[clinic end generated code: output=4effdb7d258e0d94 input=e1f81d4fcfd26497]*/ +/*[clinic end generated code: output=4effdb7d258e0d94 input=eb25de0757baf4a0]*/ { Py_complex y; @@ -1162,7 +1163,6 @@ cmath_isinf_impl(PyObject *module, Py_complex z) } /*[clinic input] -@permit_long_docstring_body cmath.isclose -> bool a: Py_complex @@ -1179,17 +1179,18 @@ Determine whether two complex numbers are close in value. Return True if a is close in value to b, and False otherwise. -For the values to be considered close, the difference between them must be -smaller than at least one of the tolerances. +For the values to be considered close, the difference between them must +be smaller than at least one of the tolerances. --inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is -not close to anything, even itself. inf and -inf are only close to themselves. +-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, +NaN is not close to anything, even itself. inf and -inf are only close +to themselves. [clinic start generated code]*/ static int cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, double rel_tol, double abs_tol) -/*[clinic end generated code: output=8a2486cc6e0014d1 input=0d45feea7c626f47]*/ +/*[clinic end generated code: output=8a2486cc6e0014d1 input=301b56c90d9a79de]*/ { double diff; diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 79c609f19aa4cff..ec3f58544cb00d5 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -1094,7 +1094,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); */ # define XML_MAJOR_VERSION 2 # define XML_MINOR_VERSION 8 -# define XML_MICRO_VERSION 0 +# define XML_MICRO_VERSION 1 # ifdef __cplusplus } diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 774e0b89d94c0ec..fa3692f9379510e 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_8_0" -expected_libexpat_version="2.8.0" -expected_libexpat_sha256="c7cec5f60ea3a42e7780781c6745255c19aa3dbfeeae58646b7132f88dc24780" +expected_libexpat_tag="R_2_8_1" +expected_libexpat_version="2.8.1" +expected_libexpat_sha256="a52eb72108be160e190b5cafa5bba8663f1313f2013e26060d1c18e26e31067b" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index e6842f3f0bf750b..95d346758563ab7 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* a5d18f6a50f536615ac1c70304f87d94f99cc85a86b502188952440610ccf0f8 (2.8.0+) +/* 75ef4224f81c052e9e5aeea2ac7de75357d2169ff9908e39edc08b9dc3052513 (2.8.1+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -387,6 +387,7 @@ typedef struct { int nDefaultAtts; int allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; + HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; typedef struct { @@ -3769,6 +3770,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; + if (! elementType->defaultAttsNames.parser) + hashTableInit(&(elementType->defaultAttsNames), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } @@ -7102,10 +7105,10 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (value || isId) { /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ - int i; - for (i = 0; i < type->nDefaultAtts; i++) - if (attId == type->defaultAtts[i].id) - return 1; + NAMED *const nameFound + = (NAMED *)lookup(parser, &(type->defaultAttsNames), attId->name, 0); + if (nameFound) + return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } @@ -7152,6 +7155,12 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, att->isCdata = isCdata; if (! isCdata) attId->maybeTokenized = XML_TRUE; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + if (! nameAddedOrFound) + return 0; + type->nDefaultAtts += 1; return 1; } @@ -7477,6 +7486,7 @@ dtdReset(DTD *p, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7518,6 +7528,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; + hashTableDestroy(&(e->defaultAttsNames)); if (e->allocDefaultAtts != 0) FREE(parser, e->defaultAtts); } @@ -7611,6 +7622,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, sizeof(ELEMENT_TYPE)); if (! newE) return 0; + + if (! newE->defaultAttsNames.parser) + hashTableInit(&(newE->defaultAttsNames), parser); + if (oldE->nDefaultAtts) { /* Detect and prevent integer overflow. * The preprocessor guard addresses the "always false" warning @@ -7635,8 +7650,9 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); for (i = 0; i < newE->nDefaultAtts; i++) { + const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( - oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0); + oldParser, &(newDtd->attributeIds), attributeName, 0); newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata; if (oldE->defaultAtts[i].value) { newE->defaultAtts[i].value @@ -7645,6 +7661,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, return 0; } else newE->defaultAtts[i].value = NULL; + + NAMED *const nameAddedOrFound = (NAMED *)lookup( + parser, &(newE->defaultAttsNames), attributeName, sizeof(NAMED)); + if (! nameAddedOrFound) { + return 0; + } } } @@ -8391,6 +8413,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; + if (! ret->defaultAttsNames.parser) + hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 923f6f5b56d32bb..3b0647152ceffed 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -7,7 +7,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_signal.h" // Py_NSIG #include "pycore_time.h" // _PyTime_FromSecondsObject() -#include "pycore_traceback.h" // _Py_DumpTracebackThreads +#include "pycore_traceback.h" // _Py_DumpStack() #ifdef HAVE_UNISTD_H # include <unistd.h> // _exit() #endif @@ -101,8 +101,6 @@ static int faulthandler_get_fileno(PyObject **file_ptr) { PyObject *result; - long fd_long; - int fd; PyObject *file = *file_ptr; if (file == NULL || file == Py_None) { @@ -124,7 +122,7 @@ faulthandler_get_fileno(PyObject **file_ptr) return -1; } } - fd = PyLong_AsInt(file); + int fd = PyLong_AsInt(file); if (fd == -1 && PyErr_Occurred()) return -1; if (fd < 0) { @@ -145,15 +143,16 @@ faulthandler_get_fileno(PyObject **file_ptr) return -1; } - fd = -1; + int fd; if (PyLong_Check(result)) { - fd_long = PyLong_AsLong(result); - if (0 <= fd_long && fd_long < INT_MAX) - fd = (int)fd_long; + fd = PyLong_AsInt(result); + } + else { + fd = -1; } Py_DECREF(result); - if (fd == -1) { + if (fd < 0) { PyErr_SetString(PyExc_RuntimeError, "file.fileno() is not a valid file descriptor"); Py_DECREF(file); @@ -206,14 +205,15 @@ faulthandler_dump_traceback(int fd, int all_threads, PyThreadState *tstate = PyGILState_GetThisThreadState(); if (all_threads == 1) { - (void)_Py_DumpTracebackThreads(fd, NULL, tstate, max_threads); + (void)PyUnstable_DumpTracebackThreads(fd, NULL, tstate, max_threads); } else { if (all_threads == FT_IGNORE_ALL_THREADS) { PUTS(fd, "<Cannot show all threads while the GIL is disabled>\n"); } - if (tstate != NULL) - _Py_DumpTraceback(fd, tstate); + if (tstate != NULL) { + PyUnstable_DumpTraceback(fd, tstate); + } } reentrant = 0; @@ -277,17 +277,18 @@ faulthandler_dump_traceback_py_impl(PyObject *module, PyObject *file, /* gh-128400: Accessing other thread states while they're running * isn't safe if those threads are running. */ _PyEval_StopTheWorld(interp); - errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate, max_threads); + errmsg = PyUnstable_DumpTracebackThreads(fd, NULL, tstate, max_threads); _PyEval_StartTheWorld(interp); - if (errmsg != NULL) { - PyErr_SetString(PyExc_RuntimeError, errmsg); - Py_XDECREF(file); - return NULL; - } } else { - _Py_DumpTraceback(fd, tstate); + errmsg = PyUnstable_DumpTraceback(fd, tstate); + } + if (errmsg != NULL) { + PyErr_SetString(PyExc_RuntimeError, errmsg); + Py_XDECREF(file); + return NULL; } + Py_XDECREF(file); if (PyErr_CheckSignals()) @@ -405,10 +406,8 @@ faulthandler_fatal_error(int signum) PUTS(fd, "\n\n"); } else { - char unknown_signum[23] = {0,}; - snprintf(unknown_signum, 23, "%d", signum); PUTS(fd, "Fatal Python error from unexpected signum: "); - PUTS(fd, unknown_signum); + _Py_DumpDecimal(fd, signum); PUTS(fd, "\n\n"); } @@ -711,10 +710,10 @@ faulthandler_thread(void *unused) /* Timeout => dump traceback */ assert(st == PY_LOCK_FAILURE); - (void)_Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); + (void)_Py_write_noraise(thread.fd, thread.header, thread.header_len); - errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL, - thread.max_threads); + errmsg = PyUnstable_DumpTracebackThreads(thread.fd, thread.interp, NULL, + thread.max_threads); ok = (errmsg == NULL); if (thread.exit) @@ -793,9 +792,9 @@ faulthandler.dump_traceback_later Dump the traceback of all threads in timeout seconds. -If repeat is true, the tracebacks of all threads are dumped every timeout -seconds. If exit is true, call _exit(1) which is not safe. max_threads -caps the number of threads dumped. +If repeat is true, the tracebacks of all threads are dumped every +timeout seconds. If exit is true, call _exit(1) which is not safe. +max_threads caps the number of threads dumped. [clinic start generated code]*/ static PyObject * @@ -803,7 +802,7 @@ faulthandler_dump_traceback_later_impl(PyObject *module, PyObject *timeout_obj, int repeat, PyObject *file, int exit, Py_ssize_t max_threads) -/*[clinic end generated code: output=543a0f3807113394 input=6836555ee157ddb4]*/ +/*[clinic end generated code: output=543a0f3807113394 input=32aaf7437d0928db]*/ { PyTime_t timeout, timeout_us; int fd; @@ -1222,7 +1221,7 @@ static PyObject * faulthandler__fatal_error_c_thread_impl(PyObject *module) /*[clinic end generated code: output=101bc8aaf4a5eec1 input=fbdca6fffd639a39]*/ { - long tid; + unsigned long tid; PyThread_type_lock lock; faulthandler_suppress_crash_report(); @@ -1234,7 +1233,7 @@ faulthandler__fatal_error_c_thread_impl(PyObject *module) PyThread_acquire_lock(lock, WAIT_LOCK); tid = PyThread_start_new_thread(faulthandler_fatal_error_thread, lock); - if (tid == -1) { + if (tid == PYTHREAD_INVALID_THREAD_ID) { PyThread_free_lock(lock); PyErr_SetString(PyExc_RuntimeError, "unable to start the thread"); return NULL; @@ -1347,21 +1346,6 @@ faulthandler__stack_overflow_impl(PyObject *module) #endif /* defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_SIGACTION) */ -static int -faulthandler_traverse(PyObject *module, visitproc visit, void *arg) -{ - Py_VISIT(thread.file); -#ifdef FAULTHANDLER_USER - if (user_signals != NULL) { - for (size_t signum=0; signum < Py_NSIG; signum++) - Py_VISIT(user_signals[signum].file); - } -#endif - Py_VISIT(fatal_error.file); - return 0; -} - - #ifdef MS_WINDOWS /*[clinic input] faulthandler._raise_exception @@ -1457,7 +1441,6 @@ static struct PyModuleDef module_def = { .m_name = "faulthandler", .m_doc = module_doc, .m_methods = module_methods, - .m_traverse = faulthandler_traverse, .m_slots = faulthandler_slots }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 18bddf46a7466bf..0093995441e390d 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -167,6 +167,8 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, gcstate->generations[2].threshold = threshold2; } #else + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); gcstate->young.threshold = threshold0; if (group_right_1) { gcstate->old[0].threshold = threshold1; @@ -174,6 +176,7 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, if (group_right_2) { gcstate->old[1].threshold = threshold2; } + _PyEval_StartTheWorld(interp); #endif Py_RETURN_NONE; } @@ -230,7 +233,7 @@ gc_get_count_impl(PyObject *module) gcstate->generations[2].count); #else return Py_BuildValue("(iii)", - gcstate->young.count, + _Py_atomic_load_int_relaxed(&gcstate->young.count), gcstate->old[0].count, gcstate->old[1].count); #endif @@ -326,13 +329,13 @@ gc.get_objects Return a list of objects tracked by the collector (excluding the list returned). -If generation is not None, return only the objects tracked by the collector -that are in that generation. +If generation is not None, return only the objects tracked by the +collector that are in that generation. [clinic start generated code]*/ static PyObject * gc_get_objects_impl(PyObject *module, Py_ssize_t generation) -/*[clinic end generated code: output=48b35fea4ba6cb0e input=a887f1d9924be7cf]*/ +/*[clinic end generated code: output=48b35fea4ba6cb0e input=89bca0d4a64e0135]*/ { if (PySys_Audit("gc.get_objects", "n", generation) < 0) { return NULL; @@ -440,19 +443,20 @@ gc_is_finalized_impl(PyObject *module, PyObject *obj) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary gc.freeze Freeze all current tracked objects and ignore them for future collections. -This can be used before a POSIX fork() call to make the gc copy-on-write friendly. -Note: collection before a POSIX fork() call may free pages for future allocation -which can cause copy-on-write. +This can be used before a POSIX fork() call to make the gc copy-on-write +friendly. +Note: collection before a POSIX fork() call may free pages for future +allocation which can cause copy-on-write. [clinic start generated code]*/ static PyObject * gc_freeze_impl(PyObject *module) -/*[clinic end generated code: output=502159d9cdc4c139 input=11fb59b0a75dcf3d]*/ +/*[clinic end generated code: output=502159d9cdc4c139 input=02674706fc9c0de6]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyGC_Freeze(interp); diff --git a/Modules/getpath.c b/Modules/getpath.c index 1e75993480ae363..0c80678a678dc8b 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -368,6 +368,7 @@ getpath_readlines(PyObject *Py_UNUSED(self), PyObject *args) if (!buffer) { Py_DECREF(r); fclose(fp); + PyErr_NoMemory(); return NULL; } diff --git a/Modules/getpath.py b/Modules/getpath.py index 4dceb5cdc8dfcf0..6199567bd777aa0 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -129,8 +129,7 @@ # checked by looking for the BUILDDIR_TXT file, which contains the # relative path to the platlib dir. The executable_dir value is # derived from joining the VPATH preprocessor variable to the -# directory containing pybuilddir.txt. If it is not found, the -# BUILD_LANDMARK file is found, which is part of the source tree. +# directory containing pybuilddir.txt. # prefix is then found by searching up for a file that should only # exist in the source tree, and the stdlib dir is set to prefix/Lib. @@ -177,7 +176,6 @@ if os_name == 'posix' or os_name == 'darwin': BUILDDIR_TXT = 'pybuilddir.txt' - BUILD_LANDMARK = 'Modules/Setup.local' DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] @@ -190,7 +188,6 @@ elif os_name == 'nt': BUILDDIR_TXT = 'pybuilddir.txt' - BUILD_LANDMARK = f'{VPATH}\\Modules\\Setup.local' DEFAULT_PROGRAM_NAME = f'python' STDLIB_SUBDIR = 'Lib' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}\\os.py', f'{STDLIB_SUBDIR}\\os.pyc'] @@ -513,13 +510,9 @@ def search_up(prefix, *landmarks, test=isfile): platstdlib_dir = real_executable_dir build_prefix = joinpath(real_executable_dir, VPATH) except (FileNotFoundError, PermissionError): - if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): - build_prefix = joinpath(real_executable_dir, VPATH) - if os_name == 'nt': - # QUIRK: Windows builds need platstdlib_dir to be the executable - # dir. Normally the builddir marker handles this, but in this - # case we need to correct manually. - platstdlib_dir = real_executable_dir + # We used to check for an alternate landmark here, but now we require + # BUILDDIR_TXT to exist. (gh-151544; CVE-2026-12003) + pass if build_prefix: if os_name == 'nt': diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index b39a8f99ed91e82..0f9eca2f73bd0c5 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -942,20 +942,19 @@ _hmac_HMAC_digest_impl(HMACObject *self) /*[clinic input] @permit_long_summary -@permit_long_docstring_body _hmac.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. -This may be used to exchange the value safely in email or other non-binary -environments. +This may be used to exchange the value safely in email or other +non-binary environments. This method may raise a MemoryError. [clinic start generated code]*/ static PyObject * _hmac_HMAC_hexdigest_impl(HMACObject *self) -/*[clinic end generated code: output=6659807a09ae14ec input=6e0e796e38d82fc8]*/ +/*[clinic end generated code: output=6659807a09ae14ec input=9097dce732ed808f]*/ { assert(self->digest_size <= Py_hmac_hash_max_digest_size); uint8_t digest[Py_hmac_hash_max_digest_size]; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index a6bfa78a461bb05..0dd31dfbc5a3469 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -107,6 +107,7 @@ typedef struct { #define batchedobject_CAST(op) ((batchedobject *)(op)) /*[clinic input] +@permit_long_summary @classmethod itertools.batched.__new__ as batched_new iterable: object @@ -136,7 +137,7 @@ than n. static PyObject * batched_new_impl(PyTypeObject *type, PyObject *iterable, Py_ssize_t n, int strict) -/*[clinic end generated code: output=c6de11b061529d3e input=7814b47e222f5467]*/ +/*[clinic end generated code: output=c6de11b061529d3e input=b31d8be8e8577a34]*/ { PyObject *it; batchedobject *bo; @@ -437,6 +438,7 @@ typedef struct { static PyObject *_grouper_create(groupbyobject *, PyObject *); /*[clinic input] +@permit_long_summary @classmethod itertools.groupby.__new__ @@ -452,7 +454,7 @@ make an iterator that returns consecutive keys and groups from the iterable static PyObject * itertools_groupby_impl(PyTypeObject *type, PyObject *it, PyObject *keyfunc) -/*[clinic end generated code: output=cbb1ae3a90fd4141 input=6b3d123e87ff65a1]*/ +/*[clinic end generated code: output=cbb1ae3a90fd4141 input=9f89fe625b20ef1a]*/ { groupbyobject *gbo; @@ -1950,10 +1952,14 @@ Return a chain object whose .__next__() method returns elements from the\n\ first iterable until it is exhausted, then elements from the next\n\ iterable, until all of the iterables are exhausted."); +PyDoc_STRVAR(chain_class_getitem_doc, +"chain is generic over the type of its contents.\n\ +This is the union of the types of the input iterable contents."); + static PyMethodDef chain_methods[] = { ITERTOOLS_CHAIN_FROM_ITERABLE_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, chain_class_getitem_doc}, {NULL, NULL} /* sentinel */ }; @@ -3163,13 +3169,13 @@ itertools.compress.__new__ selectors as seq2: object Return data elements corresponding to true selector elements. -Forms a shorter iterator from selected data elements using the selectors to -choose the data elements. +Forms a shorter iterator from selected data elements using the selectors +to choose the data elements. [clinic start generated code]*/ static PyObject * itertools_compress_impl(PyTypeObject *type, PyObject *seq1, PyObject *seq2) -/*[clinic end generated code: output=7e67157212ed09e0 input=79596d7cd20c77e5]*/ +/*[clinic end generated code: output=7e67157212ed09e0 input=32ca4347dbc46749]*/ { PyObject *data=NULL, *selectors=NULL; compressobject *lz; @@ -3427,6 +3433,7 @@ slow_mode: when cnt == PY_SSIZE_T_MAX, step is not int(1), or cnt is a float. */ /*[clinic input] +@permit_long_summary @classmethod itertools.count.__new__ start as long_cnt: object(c_default="NULL") = 0 @@ -3444,7 +3451,7 @@ Equivalent to: static PyObject * itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, PyObject *long_step) -/*[clinic end generated code: output=09a9250aebd00b1c input=d7a85eec18bfcd94]*/ +/*[clinic end generated code: output=09a9250aebd00b1c input=91e4b12c0e88b9f4]*/ { countobject *lz; int fast_mode; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 6b7fc004d0d858f..64e5372d73d2f29 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -187,13 +187,6 @@ tl_to_d(TripleLength total) } -/* - sin(pi*x), giving accurate results for all finite x (especially x - integral or close to an integer). This is here for use in the - reflection formula for the gamma function. It conforms to IEEE - 754-2008 for finite arguments, but not for infinities or nans. -*/ - static const double pi = 3.141592653589793238462643383279502884197; static const double logpi = 1.144729885849400174143427351353058711647; @@ -219,13 +212,120 @@ static const double logpi = 1.144729885849400174143427351353058711647; } \ } +#ifndef HAVE_ACOSPI +/* + acos(x)/pi. It conforms to C23 Annex 'F'. +*/ + +static double +m_acospi(double x) +{ + double r = acos(x)/pi; + if (isgreater(r, 1.0)) { + return 1.0; + } + return r; +} +#else +#define m_acospi acospi +#endif + +#ifndef HAVE_ASINPI +/* + asin(x)/pi. It conforms to C23 Annex 'F'. +*/ + +static double +m_asinpi(double x) +{ + double r = asin(x)/pi; + if (isgreater(fabs(r), 0.5)) { + return copysign(0.5, r); + } + return r; +} +#else +#define m_asinpi asinpi +#endif + +#ifndef HAVE_ATANPI +/* + asin(x)/pi. It conforms to C23 Annex 'F'. +*/ + +static double +m_atanpi(double x) +{ + double r = atan(x)/pi; + if (isgreater(fabs(r), 0.5)) { + return copysign(0.5, r); + } + return r; +} +#else +#define m_atanpi atanpi +#endif + +#ifndef HAVE_ATAN2PI +/* + asin(x)/pi. It conforms to C23 Annex 'F'. +*/ + +static double +m_atan2pi(double y, double x) +{ + double r = atan2(y, x)/pi; + if (isgreater(fabs(r), 1.0)) { + return copysign(1.0, r); + } + return r; +} +#else +#define m_atan2pi atan2pi +#endif + +#ifndef HAVE_COSPI +/* + cos(pi*x), giving accurate results for all finite x (especially x + integral or close to an integer). It conforms to C23 Annex 'F'. +*/ + +static double +m_cospi(double x) +{ + if (!isfinite(x)) { + return cos(x); + } + x = fabs(x - 2.0 * round(0.5 * x)); + if (x <= 0.25) { + return cos(pi * x); + } + if (x == 0.5) { + return 0.0; + } + if (x <= 0.75) { + return sin(pi * (0.5 - x)); + } + return -cos(pi * (1.0 - x)); +} +#else +#define m_cospi cospi +#endif + +#ifndef HAVE_SINPI +/* + sin(pi*x), giving accurate results for all finite x (especially x + integral or close to an integer). It conforms to C23 Annex 'F'. +*/ + static double m_sinpi(double x) { double y, r; int n; - /* this function should only ever be called for finite arguments */ - assert(isfinite(x)); + if (!isfinite(x)) { + return sin(x); + } y = fmod(fabs(x), 2.0); n = (int)round(2.0*y); assert(0 <= n && n <= 4); @@ -252,6 +352,47 @@ m_sinpi(double x) } return copysign(1.0, x)*r; } +#else +#define m_sinpi sinpi +#endif + +#ifndef HAVE_TANPI +/* + tan(pi*x), giving accurate results for all finite x (especially x + integral or close to an integer). It conforms to C23 Annex 'F'. +*/ + +static double +m_tanpi(double x) +{ + double y, absy; + if (!isfinite(x)) { + return tan(x); + } + y = x - 2.0 * round(0.5 * x); + absy = fabs(y); + if (absy == 0.0) { + return copysign(0.0, x); + } + if (absy == 1.0) { + return copysign(0.0, -x); + } + if (absy == 0.5) { + errno = ERANGE; + return 1.0 / copysign(0.0, y); + } + if (absy > 0.5) { + y -= copysign(1.0, y); + absy = fabs(y); + } + if (absy <= 0.25) { + return tan(pi * y); + } + return copysign(1.0 / tan(pi * (0.5 - absy)), y); +} +#else +#define m_tanpi tanpi +#endif /* Implementation of the real gamma function. Kept here to work around issues (see e.g. gh-70309) with quality of libm's tgamma/lgamma implementations @@ -952,6 +1093,11 @@ FUNC1D(acosh, acosh, 0, "acosh($module, x, /)\n--\n\n" "Return the inverse hyperbolic cosine of x.", "expected argument value not less than 1, got %s") +FUNC1D(acospi, m_acospi, 0, + "acospi($module, x, /)\n--\n\n" + "Return the arc cosine (measured in half-turns) of x.\n\n" + "The result is between 0 and 1.", + "expected a number in range from -1 up to 1, got %s") FUNC1D(asin, asin, 0, "asin($module, x, /)\n--\n\n" "Return the arc sine (measured in radians) of x.\n\n" @@ -960,6 +1106,11 @@ FUNC1D(asin, asin, 0, FUNC1(asinh, asinh, 0, "asinh($module, x, /)\n--\n\n" "Return the inverse hyperbolic sine of x.") +FUNC1D(asinpi, m_asinpi, 0, + "asinpi($module, x, /)\n--\n\n" + "Return the arc sine (measured in half-turns) of x.\n\n" + "The result is between -1/2 and 1/2.", + "expected a number in range from -1 up to 1, got %s") FUNC1(atan, atan, 0, "atan($module, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of x.\n\n" @@ -968,10 +1119,19 @@ FUNC2(atan2, atan2, "atan2($module, y, x, /)\n--\n\n" "Return the arc tangent (measured in radians) of y/x.\n\n" "Unlike atan(y/x), the signs of both x and y are considered.") +FUNC2(atan2pi, m_atan2pi, + "atan2pi($module, y, x, /)\n--\n\n" + "Return the arc tangent (measured in half-turns) of y/x.\n\n" + "Unlike atanpi(y/x), the signs of both x and y are considered.") FUNC1D(atanh, atanh, 0, "atanh($module, x, /)\n--\n\n" "Return the inverse hyperbolic tangent of x.", "expected a number between -1 and 1, got %s") +FUNC1D(atanpi, m_atanpi, 0, + "atanpi($module, x, /)\n--\n\n" + "Return the arc tangent (measured in half-turns) of x.\n\n" + "The result is between 0 and 1.", + "expected a number in range from -1 up to 1, got %s") FUNC1(cbrt, cbrt, 0, "cbrt($module, x, /)\n--\n\n" "Return the cube root of x.") @@ -1024,6 +1184,10 @@ FUNC1D(cos, cos, 0, FUNC1(cosh, cosh, 1, "cosh($module, x, /)\n--\n\n" "Return the hyperbolic cosine of x.") +FUNC1D(cospi, m_cospi, 0, + "cospi($module, x, /)\n--\n\n" + "Return the cosine of x (measured in half-turns).", + "expected a finite input, got %s") FUNC1A(erf, erf, "erf($module, x, /)\n--\n\n" "Error function at x.") @@ -1158,6 +1322,10 @@ FUNC1D(sin, sin, 0, FUNC1(sinh, sinh, 1, "sinh($module, x, /)\n--\n\n" "Return the hyperbolic sine of x.") +FUNC1D(sinpi, m_sinpi, 0, + "sinpi($module, x, /)\n--\n\n" + "Return the sine of x (measured in half-turns).", + "expected a finite input, got %s") FUNC1D(sqrt, sqrt, 0, "sqrt($module, x, /)\n--\n\n" "Return the square root of x.", @@ -1169,6 +1337,10 @@ FUNC1D(tan, tan, 0, FUNC1(tanh, tanh, 0, "tanh($module, x, /)\n--\n\n" "Return the hyperbolic tangent of x.") +FUNC1D(tanpi, m_tanpi, 1, + "tanpi($module, x, /)\n--\n\n" + "Return the tangent of x (measured in half-turns).", + "expected a finite input, got %s") /* Precision summation function as msum() by Raymond Hettinger in <https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/>, @@ -1438,13 +1610,14 @@ math.frexp Return the mantissa and exponent of x, as pair (m, e). -m is a float and e is an int, such that x = m * 2.**e. -If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0. +If x is a finite nonzero number, then m is a float with +0.5 <= abs(m) < 1.0 and an integer e is such that +x == m * 2**e exactly. Else, return (x, 0). [clinic start generated code]*/ static PyObject * math_frexp_impl(PyObject *module, double x) -/*[clinic end generated code: output=03e30d252a15ad4a input=96251c9e208bc6e9]*/ +/*[clinic end generated code: output=03e30d252a15ad4a input=215cf8ea28a0959b]*/ { int i; /* deal with special cases directly, to sidestep platform @@ -2541,6 +2714,7 @@ math_isnan_impl(PyObject *module, double x) /*[clinic input] +@permit_long_summary math.isinf x: double @@ -2551,7 +2725,7 @@ Return True if x is a positive or negative infinity, and False otherwise. static PyObject * math_isinf_impl(PyObject *module, double x) -/*[clinic end generated code: output=9f00cbec4de7b06b input=32630e4212cf961f]*/ +/*[clinic end generated code: output=9f00cbec4de7b06b input=8584152a71a3aea9]*/ { return PyBool_FromLong((long)isinf(x)); } @@ -2831,7 +3005,7 @@ math_prod_impl(PyObject *module, PyObject *iterable, PyObject *start) /*[clinic input] -@permit_long_docstring_body +@permit_long_summary math.nextafter x: double @@ -2844,13 +3018,13 @@ Return the floating-point value the given number of steps after x towards y. If steps is not specified or is None, it defaults to 1. -Raises a TypeError, if x or y is not a double, or if steps is not an integer. -Raises ValueError if steps is negative. +Raises a TypeError, if x or y is not a double, or if steps is not +an integer. Raises ValueError if steps is negative. [clinic start generated code]*/ static PyObject * math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) -/*[clinic end generated code: output=cc6511f02afc099e input=cc8f0dad1b27a8a4]*/ +/*[clinic end generated code: output=cc6511f02afc099e input=3a9151e6b1e9f346]*/ { #if defined(_AIX) if (x == y) { @@ -3045,16 +3219,21 @@ math_exec(PyObject *module) static PyMethodDef math_methods[] = { {"acos", math_acos, METH_O, math_acos_doc}, {"acosh", math_acosh, METH_O, math_acosh_doc}, + {"acospi", math_acospi, METH_O, math_acospi_doc}, {"asin", math_asin, METH_O, math_asin_doc}, {"asinh", math_asinh, METH_O, math_asinh_doc}, + {"asinpi", math_asinpi, METH_O, math_asinpi_doc}, {"atan", math_atan, METH_O, math_atan_doc}, {"atan2", _PyCFunction_CAST(math_atan2), METH_FASTCALL, math_atan2_doc}, {"atanh", math_atanh, METH_O, math_atanh_doc}, + {"atan2pi", _PyCFunction_CAST(math_atan2pi), METH_FASTCALL, math_atan2pi_doc}, + {"atanpi", math_atanpi, METH_O, math_atanpi_doc}, {"cbrt", math_cbrt, METH_O, math_cbrt_doc}, MATH_CEIL_METHODDEF {"copysign", _PyCFunction_CAST(math_copysign), METH_FASTCALL, math_copysign_doc}, {"cos", math_cos, METH_O, math_cos_doc}, {"cosh", math_cosh, METH_O, math_cosh_doc}, + {"cospi", math_cospi, METH_O, math_cospi_doc}, MATH_DEGREES_METHODDEF MATH_DIST_METHODDEF {"erf", math_erf, METH_O, math_erf_doc}, @@ -3091,9 +3270,11 @@ static PyMethodDef math_methods[] = { MATH_SIGNBIT_METHODDEF {"sin", math_sin, METH_O, math_sin_doc}, {"sinh", math_sinh, METH_O, math_sinh_doc}, + {"sinpi", math_sinpi, METH_O, math_sinpi_doc}, {"sqrt", math_sqrt, METH_O, math_sqrt_doc}, {"tan", math_tan, METH_O, math_tan_doc}, {"tanh", math_tanh, METH_O, math_tanh_doc}, + {"tanpi", math_tanpi, METH_O, math_tanpi_doc}, MATH_SUMPROD_METHODDEF MATH_TRUNC_METHODDEF MATH_PROD_METHODDEF diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index a30afe91f8fa171..6fb04ba7bd47c67 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -620,8 +620,6 @@ mmap_gfind_lock_held(mmap_object *self, Py_buffer *view, PyObject *start_obj, start += self->size; if (start < 0) start = 0; - else if (start > self->size) - start = self->size; if (end < 0) end += self->size; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 51aee5afd35b6da..255576cc057cdd4 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -885,13 +885,14 @@ _overlapped.Overlapped.getresult Retrieve result of operation. -If wait is true then it blocks until the operation is finished. If wait -is false and the operation is still pending then an error is raised. +If wait is true then it blocks until the operation is finished. If +wait is false and the operation is still pending then an error is +raised. [clinic start generated code]*/ static PyObject * _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) -/*[clinic end generated code: output=8c9bd04d08994f6c input=aa5b03e9897ca074]*/ +/*[clinic end generated code: output=8c9bd04d08994f6c input=852fbd817cbd2b3d]*/ { DWORD transferred = 0; BOOL ret; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5bd53c2146a822f..c52d2e7d5bfbd9f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2365,7 +2365,7 @@ PyDoc_STRVAR(stat_result__doc__, "stat_result: Result from stat, fstat, or lstat.\n\n\ This object may be accessed either as a tuple of\n\ (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)\n\ -or via the attributes st_mode, st_ino, st_dev, st_nlink, st_uid, and so on.\n\ +or via the attributes st_mode, st_ino, st_dev, st_nlink, and so on.\n\ \n\ Posix/windows: If your platform supports st_blksize, st_blocks, st_rdev,\n\ or st_flags, they are available as attributes only.\n\ @@ -3123,7 +3123,7 @@ class path_t_converter(CConverter): impl_by_reference = True parse_by_reference = True default_type = () - c_init_default = "<placeholder>" # overridden in pre_render(() + c_init_default = "<placeholder>" # overridden in pre_render() converter = 'path_converter' @@ -3266,7 +3266,7 @@ class confname_converter(CConverter): """, argname=argname, converter=self.converter, table=self.table) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=d58f18bdf3bd3565]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=ddbf3ac90a981122]*/ /*[clinic input] @@ -3925,14 +3925,14 @@ os.chdir Change the current working directory to the specified path. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. -If this functionality is unavailable, using it raises an exception. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. [clinic start generated code]*/ static PyObject * os_chdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=3be6400eee26eaae input=a74ceab5d72adf74]*/ +/*[clinic end generated code: output=3be6400eee26eaae input=64673c342e4369f1]*/ { int result; @@ -4045,15 +4045,16 @@ win32_fchmod(int fd, int mode) os.chmod path: path_t(allow_fd='PATH_HAVE_FCHMOD') - Path to be modified. May always be specified as a str, bytes, or a path-like object. - On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. + Path to be modified. May always be specified as a str, bytes, or + a path-like object. On some platforms, path may also be specified + as an open file descriptor. If this functionality is unavailable, + using it raises an exception. mode: int Operating-system mode bitfield. - Be careful when using number literals for *mode*. The conventional UNIX notation for - numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in - Python. + Be careful when using number literals for *mode*. The conventional + UNIX notation for numeric modes uses an octal base, which needs to + be indicated with a ``0o`` prefix in Python. * @@ -4080,7 +4081,7 @@ dir_fd and follow_symlinks may not be implemented on your platform. static PyObject * os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=5cf6a94915cc7bff input=fcf115d174b9f3d8]*/ +/*[clinic end generated code: output=5cf6a94915cc7bff input=7b6e2eeadd8bf199]*/ { int result; @@ -4216,9 +4217,9 @@ os.fchmod The file descriptor of the file to be modified. mode: int Operating-system mode bitfield. - Be careful when using number literals for *mode*. The conventional UNIX notation for - numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in - Python. + Be careful when using number literals for *mode*. The conventional + UNIX notation for numeric modes uses an octal base, which needs to + be indicated with a ``0o`` prefix in Python. Change the access permissions of the file given by file descriptor fd. @@ -4227,7 +4228,7 @@ Equivalent to os.chmod(fd, mode). static PyObject * os_fchmod_impl(PyObject *module, int fd, int mode) -/*[clinic end generated code: output=afd9bc05b4e426b3 input=b5594618bbbc22df]*/ +/*[clinic end generated code: output=afd9bc05b4e426b3 input=d24331f9fdc17f49]*/ { int res; @@ -4261,6 +4262,7 @@ os_fchmod_impl(PyObject *module, int fd, int mode) #if defined(HAVE_LCHMOD) || defined(MS_WINDOWS) /*[clinic input] +@permit_long_summary os.lchmod path: path_t @@ -4268,13 +4270,13 @@ os.lchmod Change the access permissions of a file, without following symbolic links. -If path is a symlink, this affects the link itself rather than the target. -Equivalent to chmod(path, mode, follow_symlinks=False)." +If path is a symlink, this affects the link itself rather than the +target. Equivalent to chmod(path, mode, follow_symlinks=False). [clinic start generated code]*/ static PyObject * os_lchmod_impl(PyObject *module, path_t *path, int mode) -/*[clinic end generated code: output=082344022b51a1d5 input=90c5663c7465d24f]*/ +/*[clinic end generated code: output=082344022b51a1d5 input=13110fb62911b015]*/ { int res; if (PySys_Audit("os.chmod", "Oii", path->object, mode, -1) < 0) { @@ -4312,9 +4314,9 @@ os.chflags Set file flags. -If follow_symlinks is False, and the last element of the path is a symbolic - link, chflags will change flags on the symbolic link itself instead of the - file the link points to. +If follow_symlinks is False, and the last element of the path is +a symbolic link, chflags() will change flags on the symbolic link itself +instead of the file the link points to. follow_symlinks may not be implemented on your platform. If it is unavailable, using it will raise a NotImplementedError. @@ -4323,7 +4325,7 @@ unavailable, using it will raise a NotImplementedError. static PyObject * os_chflags_impl(PyObject *module, path_t *path, unsigned long flags, int follow_symlinks) -/*[clinic end generated code: output=85571c6737661ce9 input=0327e29feb876236]*/ +/*[clinic end generated code: output=85571c6737661ce9 input=31391927707be1de]*/ { int result; @@ -4473,7 +4475,8 @@ os_fdatasync_impl(PyObject *module, int fd) os.chown path : path_t(allow_fd='PATH_HAVE_FCHOWN') - Path to be examined; can be string, bytes, a path-like object, or open-file-descriptor int. + Path to be examined; can be string, bytes, a path-like object, or + open-file-descriptor int. uid: uid_t @@ -4481,7 +4484,7 @@ os.chown * - dir_fd : dir_fd(requires='fchownat') = None + dir_fd: dir_fd(requires='fchownat') = None If not None, it should be a file descriptor open to a directory, and path should be relative; path will then be relative to that directory. @@ -4491,27 +4494,28 @@ os.chown stat will examine the symbolic link itself instead of the file the link points to. -Change the owner and group id of path to the numeric uid and gid.\ +Change the owner and group id of path to the numeric uid and gid. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, chown will modify the symbolic link itself instead of the file the - link points to. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, chown will modify the symbolic link itself instead of +the file the link points to. It is an error to use dir_fd or follow_symlinks when specifying path as - an open file descriptor. -dir_fd and follow_symlinks may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +an open file descriptor. +dir_fd and follow_symlinks may not be implemented on your platform. If +they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=4beadab0db5f70cd input=b08c5ec67996a97d]*/ +/*[clinic end generated code: output=4beadab0db5f70cd input=509c91b7a0e72f52]*/ { int result; @@ -4795,7 +4799,6 @@ os_getcwdb_impl(PyObject *module) #ifdef HAVE_LINK /*[clinic input] -@permit_long_docstring_body os.link src : path_t @@ -4808,20 +4811,21 @@ os.link Create a hard link to a file. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. If follow_symlinks is False, and the last element of src is a symbolic - link, link will create a link to the symbolic link itself instead of the - file the link points to. -src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your - platform. If they are unavailable, using them will raise a - NotImplementedError. +link, link will create a link to the symbolic link itself instead of the +file the link points to. +src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on +your platform. If they are unavailable, using them will raise +a NotImplementedError. [clinic start generated code]*/ static PyObject * os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int follow_symlinks) -/*[clinic end generated code: output=7f00f6007fd5269a input=e2a50a6497050e44]*/ +/*[clinic end generated code: output=7f00f6007fd5269a input=a28e6866fbd20a01]*/ { #ifdef MS_WINDOWS BOOL result = FALSE; @@ -5115,30 +5119,28 @@ _posix_listdir(path_t *path, PyObject *list) /*[clinic input] -@permit_long_docstring_body os.listdir path : path_t(nullable=True, allow_fd='PATH_HAVE_FDOPENDIR') = None Return a list containing the names of the files in the directory. -path can be specified as either str, bytes, or a path-like object. If path is bytes, - the filenames returned will also be bytes; in all other circumstances - the filenames returned will be str. +path can be specified as either str, bytes, or a path-like object. If +path is bytes, the filenames returned will also be bytes; in all other +circumstances the filenames returned will be str. If path is None, uses the path='.'. -On some platforms, path may also be specified as an open file descriptor;\ - the file descriptor must refer to a directory. - If this functionality is unavailable, using it raises NotImplementedError. +On some platforms, path may also be specified as an open file +descriptor; the file descriptor must refer to a directory. If this +functionality is unavailable, using it raises NotImplementedError. The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory. - [clinic start generated code]*/ static PyObject * os_listdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=293045673fcd1a75 input=0bd1728387391b9a]*/ +/*[clinic end generated code: output=293045673fcd1a75 input=4eefe7c6a42ec9b2]*/ { if (PySys_Audit("os.listdir", "O", path->object ? path->object : Py_None) < 0) { @@ -5597,6 +5599,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) } /*[clinic input] +@permit_long_summary os._findfirstfile path: path_t / @@ -5605,7 +5608,7 @@ A function to get the real file name without accessing the file in Windows. static PyObject * os__findfirstfile_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=106dd3f0779c83dd input=0734dff70f60e1a8]*/ +/*[clinic end generated code: output=106dd3f0779c83dd input=48c319aaa48d05d4]*/ { PyObject *result; HANDLE hFindFile; @@ -5696,7 +5699,7 @@ os__path_splitroot_impl(PyObject *module, path_t *path) buffer = (wchar_t*)PyMem_Malloc(sizeof(wchar_t) * (wcslen(path->wide) + 1)); if (!buffer) { - return NULL; + return PyErr_NoMemory(); } wcscpy(buffer, path->wide); for (wchar_t *p = wcschr(buffer, L'/'); p; p = wcschr(p, L'/')) { @@ -6142,6 +6145,9 @@ os__path_normpath_impl(PyObject *module, path_t *path) else { result = PyUnicode_FromWideChar(norm_path, norm_len); } + if (result == NULL) { + return NULL; + } if (PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } @@ -6159,22 +6165,21 @@ os.mkdir dir_fd : dir_fd(requires='mkdirat') = None -# "mkdir(path, mode=0o777, *, dir_fd=None)\n\n\ - Create a directory. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. -The mode argument is ignored on Windows. Where it is used, the current umask -value is first masked out. +The mode argument is ignored on Windows. Where it is used, the current +umask value is first masked out. [clinic start generated code]*/ static PyObject * os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) -/*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ +/*[clinic end generated code: output=a70446903abe821f input=30270d369599634b]*/ { int result; #ifdef MS_WINDOWS @@ -6428,7 +6433,6 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is /*[clinic input] -@permit_long_docstring_body os.rename src : path_t @@ -6440,38 +6444,39 @@ os.rename Rename a file or directory. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. src_dir_fd and dst_dir_fd, may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_rename_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=59e803072cf41230 input=11aae8c091162766]*/ +/*[clinic end generated code: output=59e803072cf41230 input=7d320d687c715fd6]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 0); } /*[clinic input] -@permit_long_docstring_body os.replace = os.rename Rename a file or directory, overwriting the destination. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. src_dir_fd and dst_dir_fd, may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_replace_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=1968c02e7857422b input=78d6c8087e90994c]*/ +/*[clinic end generated code: output=1968c02e7857422b input=44ed6b762d5953fc]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 1); } @@ -6486,15 +6491,16 @@ os.rmdir Remove a directory. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=080eb54f506e8301 input=38c8b375ca34a7e2]*/ +/*[clinic end generated code: output=080eb54f506e8301 input=84325211e33a98e0]*/ { int result; #ifdef HAVE_UNLINKAT @@ -6665,16 +6671,17 @@ os.unlink Remove a file (same as remove()). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_unlink_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=621797807b9963b1 input=d7bcde2b1b2a2552]*/ +/*[clinic end generated code: output=621797807b9963b1 input=1a2ef2579207eab1]*/ { int result; #ifdef HAVE_UNLINKAT @@ -6726,15 +6733,16 @@ os.remove = os.unlink Remove a file (same as unlink()). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_remove_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=a8535b28f0068883 input=e05c5ab55cd30983]*/ +/*[clinic end generated code: output=a8535b28f0068883 input=9f6e66912126bd56]*/ { return os_unlink_impl(module, path, dir_fd); } @@ -7056,38 +7064,37 @@ os.utime dir_fd: dir_fd(requires='futimensat') = None follow_symlinks: bool=True -# "utime(path, times=None, *[, ns], dir_fd=None, follow_symlinks=True)\n\ - Set the access and modified time of path. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. If times is not None, it must be a tuple (atime, mtime); - atime and mtime should be expressed as float seconds since the epoch. +atime and mtime should be expressed as float seconds since the epoch. If ns is specified, it must be a tuple (atime_ns, mtime_ns); - atime_ns and mtime_ns should be expressed as integer nanoseconds - since the epoch. +atime_ns and mtime_ns should be expressed as integer nanoseconds +since the epoch. If times is None and ns is unspecified, utime uses the current time. Specifying tuples for both times and ns is an error. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, utime will modify the symbolic link itself instead of the file the - link points to. -It is an error to use dir_fd or follow_symlinks when specifying path - as an open file descriptor. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, utime will modify the symbolic link itself instead of +the file the link points to. +It is an error to use dir_fd or follow_symlinks when specifying path as +an open file descriptor. dir_fd and follow_symlinks may not be available on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=cfcac69d027b82cf input=2fbd62a2f228f8f4]*/ +/*[clinic end generated code: output=cfcac69d027b82cf input=5ab470b2bc250788]*/ { #ifdef MS_WINDOWS HANDLE hFile; @@ -7244,6 +7251,7 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, /*[clinic input] +@permit_long_summary os._exit status: int @@ -7253,7 +7261,7 @@ Exit to the system with specified status, without normal exit processing. static PyObject * os__exit_impl(PyObject *module, int status) -/*[clinic end generated code: output=116e52d9c2260d54 input=5e6d57556b0c4a62]*/ +/*[clinic end generated code: output=116e52d9c2260d54 input=c35d282acfebe8fd]*/ { _exit(status); return NULL; /* Make gcc -Wall happy */ @@ -8057,7 +8065,8 @@ os.posix_spawn resetids: bool = False If the value is `true` the POSIX_SPAWN_RESETIDS will be activated. setsid: bool = False - If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. + If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP + will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. setsigdef: object(c_default='NULL') = () @@ -8074,7 +8083,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=14a1098c566bc675 input=69e7c9ebbdcf94a5]*/ +/*[clinic end generated code: output=14a1098c566bc675 input=c7592dcbc96e8114]*/ { return py_posix_spawn(0, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -8103,7 +8112,8 @@ os.posix_spawnp resetids: bool = False If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. setsid: bool = False - If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. + If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP + will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. setsigdef: object(c_default='NULL') = () @@ -8120,7 +8130,7 @@ os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=7b9aaefe3031238d input=a5c057527c6881a5]*/ +/*[clinic end generated code: output=7b9aaefe3031238d input=43ccc1452cae2be3]*/ { return py_posix_spawn(1, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -9156,13 +9166,13 @@ os.posix_openpt -> int Open and return a file descriptor for a master pseudo-terminal device. Performs a posix_openpt() C function call. The oflag argument is used to -set file status flags and file access modes as specified in the manual page -of posix_openpt() of your system. +set file status flags and file access modes as specified in the manual +page of posix_openpt() of your system. [clinic start generated code]*/ static int os_posix_openpt_impl(PyObject *module, int oflag) -/*[clinic end generated code: output=ee0bc2624305fc79 input=0de33d0e29693caa]*/ +/*[clinic end generated code: output=ee0bc2624305fc79 input=3ce4eb297fa64307]*/ { int fd; @@ -9437,6 +9447,7 @@ os_openpty_impl(PyObject *module) #if defined(HAVE_LOGIN_TTY) || defined(HAVE_FALLBACK_LOGIN_TTY) /*[clinic input] +@permit_long_summary os.login_tty fd: fildes @@ -9451,7 +9462,7 @@ calling process; close fd. static PyObject * os_login_tty_impl(PyObject *module, int fd) -/*[clinic end generated code: output=495a79911b4cc1bc input=5f298565099903a2]*/ +/*[clinic end generated code: output=495a79911b4cc1bc input=b102a7c36e8baf00]*/ { #ifdef HAVE_LOGIN_TTY if (login_tty(fd) == -1) { @@ -9806,14 +9817,14 @@ os.initgroups Initialize the group access list. -Call the system initgroups() to initialize the group access list with all of -the groups of which the specified username is a member, plus the specified -group id. +Call the system initgroups() to initialize the group access list with +all of the groups of which the specified username is a member, plus the +specified group id. [clinic start generated code]*/ static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, int gid) -/*[clinic end generated code: output=7f074d30a425fd3a input=984e60c7fed88cb4]*/ +/*[clinic end generated code: output=7f074d30a425fd3a input=35f2d4fb7fcc0bdf]*/ #else /*[clinic input] os.initgroups @@ -9824,14 +9835,14 @@ os.initgroups Initialize the group access list. -Call the system initgroups() to initialize the group access list with all of -the groups of which the specified username is a member, plus the specified -group id. +Call the system initgroups() to initialize the group access list with +all of the groups of which the specified username is a member, plus the +specified group id. [clinic start generated code]*/ static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, gid_t gid) -/*[clinic end generated code: output=59341244521a9e3f input=17d8fbe2dea42ca4]*/ +/*[clinic end generated code: output=59341244521a9e3f input=7e4514dff4526a95]*/ #endif { const char *username = PyBytes_AS_STRING(oname); @@ -10028,12 +10039,13 @@ os.getppid Return the parent's process id. If the parent process has already exited, Windows machines will still -return its id; others systems will return the id of the 'init' process (1). +return its id; others systems will return the id of the 'init' proces +(1). [clinic start generated code]*/ static PyObject * os_getppid_impl(PyObject *module) -/*[clinic end generated code: output=43b2a946a8c603b4 input=e637cb87539c030e]*/ +/*[clinic end generated code: output=43b2a946a8c603b4 input=e17c1de18f41316b]*/ { #ifdef MS_WINDOWS return win32_getppid(); @@ -10587,13 +10599,13 @@ os.waitid Returns the result of waiting for a process or processes. -Returns either waitid_result or None if WNOHANG is specified and there are -no children in a waitable state. +Returns either waitid_result or None if WNOHANG is specified and there +are no children in a waitable state. [clinic start generated code]*/ static PyObject * os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) -/*[clinic end generated code: output=5d2e1c0bde61f4d8 input=d8e7f76e052b7920]*/ +/*[clinic end generated code: output=5d2e1c0bde61f4d8 input=14956bc8d102b5db]*/ { PyObject *result; int res; @@ -10760,13 +10772,13 @@ os.pidfd_open Return a file descriptor referring to the process *pid*. -The descriptor can be used to perform process management without races and -signals. +The descriptor can be used to perform process management without races +and signals. [clinic start generated code]*/ static PyObject * os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) -/*[clinic end generated code: output=5c7252698947dc41 input=c3fd99ce947ccfef]*/ +/*[clinic end generated code: output=5c7252698947dc41 input=03058b32c389f874]*/ { int fd = syscall(__NR_pidfd_open, pid, flags); if (fd < 0) { @@ -10777,6 +10789,35 @@ os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) #endif +#if defined(__linux__) && defined(__NR_pidfd_getfd) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) +/*[clinic input] +os.pidfd_getfd + pidfd: int + A process file descriptor. + targetfd: int + The file descriptor to duplicate from the target process. + * + flags: unsigned_int = 0 + Reserved, must be 0. + +Duplicate a file descriptor from the process referred to by *pidfd*. +[clinic start generated code]*/ + +static PyObject * +os_pidfd_getfd_impl(PyObject *module, int pidfd, int targetfd, + unsigned int flags) +/*[clinic end generated code: output=e1a1415a13c7137f input=ef6417fb10deb1cc]*/ +{ + int fd = syscall(__NR_pidfd_getfd, pidfd, targetfd, flags); + if (fd < 0) { + return posix_error(); + } + return PyLong_FromLong(fd); +} +#endif + + #ifdef HAVE_SETNS /*[clinic input] os.setns @@ -10845,8 +10886,9 @@ os.readlink Return a string representing the path to which the symbolic link points. -If dir_fd is not None, it should be a file descriptor open to a directory, -and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. dir_fd may not be implemented on your platform. If it is unavailable, using it will raise a NotImplementedError. @@ -10854,7 +10896,7 @@ using it will raise a NotImplementedError. static PyObject * os_readlink_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=d21b732a2e814030 input=113c87e0db1ecaf2]*/ +/*[clinic end generated code: output=d21b732a2e814030 input=03d10130870dbca8]*/ { #if defined(HAVE_READLINK) char buffer[MAXPATHLEN+1]; @@ -11049,26 +11091,25 @@ os.symlink * dir_fd: dir_fd(requires='symlinkat')=None -# "symlink(src, dst, target_is_directory=False, *, dir_fd=None)\n\n\ - Create a symbolic link pointing to src named dst. target_is_directory is required on Windows if the target is to be - interpreted as a directory. (On Windows, symlink requires - Windows 6.0 or greater, and raises a NotImplementedError otherwise.) - target_is_directory is ignored on non-Windows platforms. +interpreted as a directory. (On Windows, symlink requires Windows 6.0 +or greater, and raises a NotImplementedError otherwise.) +target_is_directory is ignored on non-Windows platforms. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_symlink_impl(PyObject *module, path_t *src, path_t *dst, int target_is_directory, int dir_fd) -/*[clinic end generated code: output=08ca9f3f3cf960f6 input=e820ec4472547bc3]*/ +/*[clinic end generated code: output=08ca9f3f3cf960f6 input=71b75467b31c45f7]*/ { #ifdef MS_WINDOWS DWORD result; @@ -11627,19 +11668,18 @@ os.open -> int * dir_fd: dir_fd(requires='openat') = None -# "open(path, flags, mode=0o777, *, dir_fd=None)\n\n\ - Open a file for low level IO. Returns a file descriptor (integer). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static int os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) -/*[clinic end generated code: output=abc7227888c8bc73 input=ad8623b29acd2934]*/ +/*[clinic end generated code: output=abc7227888c8bc73 input=75f7b4eaf92f2225]*/ { int fd; int async_err = 0; @@ -11913,7 +11953,6 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) /*[clinic input] -@permit_long_docstring_body os.lseek -> Py_off_t fd: int @@ -11929,12 +11968,13 @@ os.lseek -> Py_off_t Set the position of a file descriptor. Return the new position. -The return value is the number of bytes relative to the beginning of the file. +The return value is the number of bytes relative to the beginning of +the file. [clinic start generated code]*/ static Py_off_t os_lseek_impl(PyObject *module, int fd, Py_off_t position, int how) -/*[clinic end generated code: output=971e1efb6b30bd2f input=4a3de549f07e1c40]*/ +/*[clinic end generated code: output=971e1efb6b30bd2f input=32ea0788da7cb44b]*/ { Py_off_t result; @@ -11998,7 +12038,6 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) } /*[clinic input] -@permit_long_docstring_body os.readinto -> Py_ssize_t fd: int buffer: Py_buffer(accept={rwbuffer}) @@ -12006,20 +12045,20 @@ os.readinto -> Py_ssize_t Read into a buffer object from a file descriptor. -The buffer should be mutable and bytes-like. On success, returns the number of -bytes read. Less bytes may be read than the size of the buffer. The underlying -system call will be retried when interrupted by a signal, unless the signal -handler raises an exception. Other errors will not be retried and an error will -be raised. +The buffer should be mutable and bytes-like. On success, returns the +number of bytes read. Less bytes may be read than the size of the +buffer. The underlying system call will be retried when interrupted by +a signal, unless the signal handler raises an exception. Other errors +will not be retried and an error will be raised. -Returns 0 if *fd* is at end of file or if the provided *buffer* has length 0 -(which can be used to check for errors without reading data). Never returns -negative. +Returns 0 if *fd* is at end of file or if the provided *buffer* has +length 0 (which can be used to check for errors without reading data). +Never returns negative. [clinic start generated code]*/ static Py_ssize_t os_readinto_impl(PyObject *module, int fd, Py_buffer *buffer) -/*[clinic end generated code: output=8091a3513c683a80 input=a770382bd3d32f9a]*/ +/*[clinic end generated code: output=8091a3513c683a80 input=2a5f8b212cb5730c]*/ { assert(buffer->len >= 0); Py_ssize_t result = _Py_read(fd, buffer->buf, buffer->len); @@ -12203,7 +12242,7 @@ os_pread_impl(PyObject *module, int fd, Py_ssize_t length, Py_off_t offset) #if defined(HAVE_PREADV) || defined (HAVE_PREADV2) /*[clinic input] -@permit_long_docstring_body +@permit_long_summary os.preadv -> Py_ssize_t fd: int @@ -12214,14 +12253,15 @@ os.preadv -> Py_ssize_t Reads from a file descriptor into a number of mutable bytes-like objects. -Combines the functionality of readv() and pread(). As readv(), it will -transfer data into each buffer until it is full and then move on to the next -buffer in the sequence to hold the rest of the data. Its fourth argument, -specifies the file offset at which the input operation is to be performed. It -will return the total number of bytes read (which can be less than the total -capacity of all the objects). +Combines the functionality of readv() and pread(). As readv(), it will +transfer data into each buffer until it is full and then move on to the +next buffer in the sequence to hold the rest of the data. Its fourth +argument, specifies the file offset at which the input operation is to +be performed. It will return the total number of bytes read (which can +be less than the total capacity of all the objects). -The flags argument contains a bitwise OR of zero or more of the following flags: +The flags argument contains a bitwise OR of zero or more of the +following flags: - RWF_HIPRI - RWF_NOWAIT @@ -12233,7 +12273,7 @@ Using non-zero flags requires Linux 4.6 or newer. static Py_ssize_t os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=26fc9c6e58e7ada5 input=34fb3b9ca06f7ba7]*/ +/*[clinic end generated code: output=26fc9c6e58e7ada5 input=bbc70c63b4f4e877]*/ { Py_ssize_t cnt, n; int async_err = 0; @@ -12570,6 +12610,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, #if defined(__APPLE__) /*[clinic input] +@permit_long_summary os._fcopyfile in_fd: int @@ -12582,7 +12623,7 @@ Efficiently copy content or metadata of 2 regular file descriptors (macOS). static PyObject * os__fcopyfile_impl(PyObject *module, int in_fd, int out_fd, int flags) -/*[clinic end generated code: output=c9d1a35a992e401b input=1e34638a86948795]*/ +/*[clinic end generated code: output=c9d1a35a992e401b input=80b53ad8863c9101]*/ { int ret; @@ -12771,6 +12812,7 @@ os_pipe2_impl(PyObject *module, int flags) #ifdef HAVE_WRITEV /*[clinic input] +@permit_long_summary os.writev -> Py_ssize_t fd: int buffers: object @@ -12784,7 +12826,7 @@ buffers must be a sequence of bytes-like objects. static Py_ssize_t os_writev_impl(PyObject *module, int fd, PyObject *buffers) -/*[clinic end generated code: output=56565cfac3aac15b input=5b8d17fe4189d2fe]*/ +/*[clinic end generated code: output=56565cfac3aac15b input=5771a0f0c2b326f2]*/ { Py_ssize_t cnt; Py_ssize_t result; @@ -12860,7 +12902,6 @@ os_pwrite_impl(PyObject *module, int fd, Py_buffer *buffer, Py_off_t offset) #if defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2) /*[clinic input] @permit_long_summary -@permit_long_docstring_body os.pwritev -> Py_ssize_t fd: int @@ -12871,14 +12912,16 @@ os.pwritev -> Py_ssize_t Writes the contents of bytes-like objects to a file descriptor at a given offset. -Combines the functionality of writev() and pwrite(). All buffers must be a sequence -of bytes-like objects. Buffers are processed in array order. Entire contents of first -buffer is written before proceeding to second, and so on. The operating system may -set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used. -This function writes the contents of each object to the file descriptor and returns -the total number of bytes written. +Combines the functionality of writev() and pwrite(). All buffers must be +a sequence of bytes-like objects. Buffers are processed in array order. +Entire contents of first buffer is written before proceeding to second, +and so on. The operating system may set a limit (sysconf() value +SC_IOV_MAX) on the number of buffers that can be used. +This function writes the contents of each object to the file descriptor +and returns the total number of bytes written. -The flags argument contains a bitwise OR of zero or more of the following flags: +The flags argument contains a bitwise OR of zero or more of the +following flags: - RWF_DSYNC - RWF_SYNC @@ -12892,7 +12935,7 @@ Using non-zero flags requires Linux 4.7 or newer. static Py_ssize_t os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=7de72245873f56bf]*/ +/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=b2e352a22f030e9a]*/ { Py_ssize_t cnt; Py_ssize_t result; @@ -13105,15 +13148,16 @@ os.mkfifo Create a "fifo" (a POSIX named pipe). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_mkfifo_impl(PyObject *module, path_t *path, int mode, int dir_fd) -/*[clinic end generated code: output=ce41cfad0e68c940 input=73032e98a36e0e19]*/ +/*[clinic end generated code: output=ce41cfad0e68c940 input=d2fb917c01e888d6]*/ { int result; int async_err = 0; @@ -13156,7 +13200,6 @@ os_mkfifo_impl(PyObject *module, path_t *path, int mode, int dir_fd) #if defined(HAVE_MKNOD) && defined(HAVE_MAKEDEV) /*[clinic input] -@permit_long_docstring_body os.mknod path: path_t @@ -13167,23 +13210,24 @@ os.mknod Create a node in the file system. -Create a node in the file system (file, device special file or named pipe) -at path. mode specifies both the permissions to use and the +Create a node in the file system (file, device special file or named +pipe) at path. mode specifies both the permissions to use and the type of node to be created, being combined (bitwise OR) with one of -S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set on mode, -device defines the newly created device special file (probably using -os.makedev()). Otherwise device is ignored. +S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set +on mode, device defines the newly created device special file (probably +using os.makedev()). Otherwise device is ignored. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_mknod_impl(PyObject *module, path_t *path, int mode, dev_t device, int dir_fd) -/*[clinic end generated code: output=92e55d3ca8917461 input=7121c4723d22545b]*/ +/*[clinic end generated code: output=92e55d3ca8917461 input=7d0099e85c6b4cba]*/ { int result; int async_err = 0; @@ -13352,13 +13396,14 @@ os.truncate Truncate a file, specified by path, to a specific length. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static PyObject * os_truncate_impl(PyObject *module, path_t *path, Py_off_t length) -/*[clinic end generated code: output=43009c8df5c0a12b input=77229cf0b50a9b77]*/ +/*[clinic end generated code: output=43009c8df5c0a12b input=ce33fd7808a511c4]*/ { int result; #ifdef MS_WINDOWS @@ -13401,6 +13446,7 @@ os_truncate_impl(PyObject *module, path_t *path, Py_off_t length) OSs, support was dropped in WASI preview2. */ #if defined(HAVE_POSIX_FALLOCATE) && !defined(__wasi__) /*[clinic input] +@permit_long_summary os.posix_fallocate fd: int @@ -13411,13 +13457,14 @@ os.posix_fallocate Ensure a file has allocated at least a particular number of bytes on disk. Ensure that the file specified by fd encompasses a range of bytes -starting at offset bytes from the beginning and continuing for length bytes. +starting at offset bytes from the beginning and continuing for length +bytes. [clinic start generated code]*/ static PyObject * os_posix_fallocate_impl(PyObject *module, int fd, Py_off_t offset, Py_off_t length) -/*[clinic end generated code: output=73f107139564aa9d input=d7a2ef0ab2ca52fb]*/ +/*[clinic end generated code: output=73f107139564aa9d input=c718971d18b96896]*/ { int result; int async_err = 0; @@ -13452,8 +13499,8 @@ os.posix_fadvise Announce an intention to access data in a specific pattern. -Announce an intention to access data in a specific pattern, thus allowing -the kernel to make optimizations. +Announce an intention to access data in a specific pattern, thus +allowing the kernel to make optimizations. The advice applies to the region of the file specified by fd starting at offset and continuing for length bytes. advice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL, @@ -13464,7 +13511,7 @@ POSIX_FADV_DONTNEED. static PyObject * os_posix_fadvise_impl(PyObject *module, int fd, Py_off_t offset, Py_off_t length, int advice) -/*[clinic end generated code: output=412ef4aa70c98642 input=0fbe554edc2f04b5]*/ +/*[clinic end generated code: output=412ef4aa70c98642 input=961b01a4518ef727]*/ { int result; int async_err = 0; @@ -13663,6 +13710,10 @@ static PyObject * os__clearenv_impl(PyObject *module) /*[clinic end generated code: output=2d6705d62c014b51 input=47d2fa7f323c43ca]*/ { + if (PySys_Audit("os._clearenv", NULL) < 0) { + return NULL; + } + errno = 0; int err = clearenv(); if (err) { @@ -13720,6 +13771,7 @@ os_WCOREDUMP_impl(PyObject *module, int status) #ifdef WIFCONTINUED /*[clinic input] +@permit_long_summary os.WIFCONTINUED -> bool status: int @@ -13732,7 +13784,7 @@ job control stop. static int os_WIFCONTINUED_impl(PyObject *module, int status) -/*[clinic end generated code: output=1e35295d844364bd input=e777e7d38eb25bd9]*/ +/*[clinic end generated code: output=1e35295d844364bd input=7b577845a0f8b12f]*/ { WAIT_TYPE wait_status; WAIT_STATUS_INT(wait_status) = status; @@ -13845,6 +13897,7 @@ os_WTERMSIG_impl(PyObject *module, int status) #ifdef WSTOPSIG /*[clinic input] +@permit_long_summary os.WSTOPSIG -> int status: int @@ -13854,7 +13907,7 @@ Return the signal that stopped the process that provided the status value. static int os_WSTOPSIG_impl(PyObject *module, int status) -/*[clinic end generated code: output=0ab7586396f5d82b input=46ebf1d1b293c5c1]*/ +/*[clinic end generated code: output=0ab7586396f5d82b input=4698db1a6a320433]*/ { WAIT_TYPE wait_status; WAIT_STATUS_INT(wait_status) = status; @@ -14045,13 +14098,14 @@ os.statvfs Perform a statvfs system call on the given path. path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static PyObject * os_statvfs_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=87106dd1beb8556e input=3f5c35791c669bd9]*/ +/*[clinic end generated code: output=87106dd1beb8556e input=1cfd9a4fd36f7425]*/ { int result; @@ -14100,6 +14154,7 @@ os_statvfs_impl(PyObject *module, path_t *path) #ifdef MS_WINDOWS /*[clinic input] +@permit_long_summary os._getdiskusage path: path_t @@ -14109,7 +14164,7 @@ Return disk usage statistics about the given path as a (total, free) tuple. static PyObject * os__getdiskusage_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=3bd3991f5e5c5dfb input=6af8d1b7781cc042]*/ +/*[clinic end generated code: output=3bd3991f5e5c5dfb input=aee7e38bc3e7fb08]*/ { BOOL retval; ULARGE_INTEGER _, total, free; @@ -14330,13 +14385,14 @@ os.pathconf -> long Return the configuration limit name for the file or directory path. If there is no limit, return -1. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static long os_pathconf_impl(PyObject *module, path_t *path, int name) -/*[clinic end generated code: output=5bedee35b293a089 input=6f6072f57b10c787]*/ +/*[clinic end generated code: output=5bedee35b293a089 input=e86f6eacfa006426]*/ { long limit; @@ -15141,13 +15197,13 @@ os.abort Abort the interpreter immediately. -This function 'dumps core' or otherwise fails in the hardest way possible -on the hosting operating system. This function never returns. +This function 'dumps core' or otherwise fails in the hardest way +possible on the hosting operating system. This function never returns. [clinic start generated code]*/ static PyObject * os_abort_impl(PyObject *module) -/*[clinic end generated code: output=dcf52586dad2467c input=cf2c7d98bc504047]*/ +/*[clinic end generated code: output=dcf52586dad2467c input=ee8bd0ed690440ab]*/ { abort(); /*NOTREACHED*/ @@ -15356,6 +15412,7 @@ os_setresgid_impl(PyObject *module, gid_t rgid, gid_t egid, gid_t sgid) #ifdef HAVE_GETRESUID /*[clinic input] +@permit_long_summary os.getresuid Return a tuple of the current process's real, effective, and saved user ids. @@ -15363,7 +15420,7 @@ Return a tuple of the current process's real, effective, and saved user ids. static PyObject * os_getresuid_impl(PyObject *module) -/*[clinic end generated code: output=8e0becff5dece5bf input=41ccfa8e1f6517ad]*/ +/*[clinic end generated code: output=8e0becff5dece5bf input=ddf95881f492cb97]*/ { uid_t ruid, euid, suid; if (getresuid(&ruid, &euid, &suid) < 0) @@ -15408,17 +15465,18 @@ os.getxattr Return the value of extended attribute attribute on path. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, getxattr will examine the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, getxattr will examine the symbolic link itself +instead of the file the link points to. [clinic start generated code]*/ static PyObject * os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, int follow_symlinks) -/*[clinic end generated code: output=5f2f44200a43cff2 input=025789491708f7eb]*/ +/*[clinic end generated code: output=5f2f44200a43cff2 input=db1021ed738d9754]*/ { if (fd_and_follow_symlinks_invalid("getxattr", path->is_fd, follow_symlinks)) return NULL; @@ -15465,7 +15523,6 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, /*[clinic input] -@permit_long_docstring_body os.setxattr path: path_t(allow_fd=True) @@ -15477,17 +15534,18 @@ os.setxattr Set extended attribute attribute on path to value. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, setxattr will modify the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, setxattr will modify the symbolic link itself instead +of the file the link points to. [clinic start generated code]*/ static PyObject * os_setxattr_impl(PyObject *module, path_t *path, path_t *attribute, Py_buffer *value, int flags, int follow_symlinks) -/*[clinic end generated code: output=98b83f63fdde26bb input=4098e6f68699f3d7]*/ +/*[clinic end generated code: output=98b83f63fdde26bb input=6c4ee6724e8947a4]*/ { ssize_t result; @@ -15530,17 +15588,18 @@ os.removexattr Remove extended attribute attribute on path. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, removexattr will modify the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, removexattr will modify the symbolic link itself +instead of the file the link points to. [clinic start generated code]*/ static PyObject * os_removexattr_impl(PyObject *module, path_t *path, path_t *attribute, int follow_symlinks) -/*[clinic end generated code: output=521a51817980cda6 input=3d9a7d36fe2f7c4e]*/ +/*[clinic end generated code: output=521a51817980cda6 input=a7ec62a86aa83f01]*/ { ssize_t result; @@ -15569,7 +15628,6 @@ os_removexattr_impl(PyObject *module, path_t *path, path_t *attribute, /*[clinic input] -@permit_long_docstring_body os.listxattr path: path_t(allow_fd=True, nullable=True) = None @@ -15578,16 +15636,17 @@ os.listxattr Return a list of extended attributes on path. -path may be either None, a string, a path-like object, or an open file descriptor. -if path is None, listxattr will examine the current directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, listxattr will examine the symbolic link itself instead of the file - the link points to. +path may be either None, a string, a path-like object, or an open file +descriptor. If path is None, listxattr will examine the current +directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, listxattr will examine the symbolic link itself instead +of the file the link points to. [clinic start generated code]*/ static PyObject * os_listxattr_impl(PyObject *module, path_t *path, int follow_symlinks) -/*[clinic end generated code: output=bebdb4e2ad0ce435 input=48aa9ac8be47dea1]*/ +/*[clinic end generated code: output=bebdb4e2ad0ce435 input=cb4a6414afaa99bd]*/ { Py_ssize_t i; PyObject *result = NULL; @@ -16490,7 +16549,7 @@ static PyMemberDef DirEntry_members[] = { {"name", Py_T_OBJECT_EX, offsetof(DirEntry, name), Py_READONLY, "the entry's base filename, relative to scandir() \"path\" argument"}, {"path", Py_T_OBJECT_EX, offsetof(DirEntry, path), Py_READONLY, - "the entry's full path name; equivalent to os.path.join(scandir_path, entry.name)"}, + "the entry's full path name; equivalent to\nos.path.join(scandir_path, entry.name)"}, {NULL} }; @@ -16505,7 +16564,7 @@ static PyMethodDef DirEntry_methods[] = { OS_DIRENTRY_INODE_METHODDEF OS_DIRENTRY___FSPATH___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("DirEntry is generic over the type of the path (str or bytes)")}, {NULL} }; @@ -16984,16 +17043,16 @@ os.scandir Return an iterator of DirEntry objects for given path. -path can be specified as either str, bytes, or a path-like object. If path -is bytes, the names of yielded DirEntry objects will also be bytes; in -all other circumstances they will be str. +path can be specified as either str, bytes, or a path-like object. If +path is bytes, the names of yielded DirEntry objects will also be bytes; +in all other circumstances they will be str. If path is None, uses the path='.'. [clinic start generated code]*/ static PyObject * os_scandir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=6eb2668b675ca89e input=6bdd312708fc3bb0]*/ +/*[clinic end generated code: output=6eb2668b675ca89e input=6ab9600993f51577]*/ { ScandirIterator *iterator; #ifdef MS_WINDOWS @@ -17140,21 +17199,20 @@ PyOS_FSPath(PyObject *path) } /*[clinic input] -@permit_long_docstring_body os.fspath path: object Return the file system path representation of the object. -If the object is str or bytes, then allow it to pass through as-is. If the -object defines __fspath__(), then return the result of that method. All other -types raise a TypeError. +If the object is str or bytes, then allow it to pass through as-is. If +the object defines __fspath__(), then return the result of that method. +All other types raise a TypeError. [clinic start generated code]*/ static PyObject * os_fspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=c3c3b78ecff2914f input=f608743e60a3211e]*/ +/*[clinic end generated code: output=c3c3b78ecff2914f input=d3c54404240d5da0]*/ { return PyOS_FSPath(path); } @@ -17331,13 +17389,13 @@ On Unix: On Windows, return status shifted right by 8 bits. On Unix, if the process is being traced or if waitpid() was called with -WUNTRACED option, the caller must first check if WIFSTOPPED(status) is true. -This function must not be called if WIFSTOPPED(status) is true. +WUNTRACED option, the caller must first check if WIFSTOPPED(status) is +true. This function must not be called if WIFSTOPPED(status) is true. [clinic start generated code]*/ static PyObject * os_waitstatus_to_exitcode_impl(PyObject *module, PyObject *status_obj) -/*[clinic end generated code: output=db50b1b0ba3c7153 input=7fe2d7fdaea3db42]*/ +/*[clinic end generated code: output=db50b1b0ba3c7153 input=3b44a23f5090006c]*/ { #ifndef MS_WINDOWS int status = PyLong_AsInt(status_obj); @@ -17606,6 +17664,7 @@ static PyMethodDef posix_methods[] = { OS_WAITID_METHODDEF OS_WAITPID_METHODDEF OS_PIDFD_OPEN_METHODDEF + OS_PIDFD_GETFD_METHODDEF OS_GETSID_METHODDEF OS_SETSID_METHODDEF OS_SETPGID_METHODDEF diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 0f0afe17513ef1c..f9e7e057f7e0f7b 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_ceval.h" // _Py_EnterRecursiveCall() +#include "pycore_codecs.h" // _PyCodec_LookupTextEncoding() #include "pycore_import.h" // _PyImport_SetModule() #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_traceback.h" // _PyTraceback_Add() @@ -393,7 +394,7 @@ my_CharacterDataHandler(void *userData, const XML_Char *data, int len) if (self->buffer == NULL) call_character_handler(self, data, len); else { - if ((self->buffer_used + len) > self->buffer_size) { + if (len > (self->buffer_size - self->buffer_used)) { if (flush_character_buffer(self) < 0) return; /* handler might have changed; drop the rest on the floor @@ -830,6 +831,7 @@ get_parse_result(pyexpat_state *state, xmlparseobject *self, int rv) #define MAX_CHUNK_SIZE (1 << 20) /*[clinic input] +@permit_long_summary pyexpat.xmlparser.SetReparseDeferralEnabled enabled: bool @@ -841,7 +843,7 @@ Enable/Disable reparse deferral; enabled by default with Expat >=2.6.0. static PyObject * pyexpat_xmlparser_SetReparseDeferralEnabled_impl(xmlparseobject *self, int enabled) -/*[clinic end generated code: output=5ec539e3b63c8c49 input=021eb9e0bafc32c5]*/ +/*[clinic end generated code: output=5ec539e3b63c8c49 input=6d3743500dcee799]*/ { #if XML_COMBINED_VERSION >= 20600 XML_SetReparseDeferralEnabled(self->itself, enabled ? XML_TRUE : XML_FALSE); @@ -1053,18 +1055,19 @@ pyexpat_xmlparser_GetBase_impl(xmlparseobject *self) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary pyexpat.xmlparser.GetInputContext Return the untranslated text of the input that caused the current event. -If the event was generated by a large amount of text (such as a start tag -for an element with many attributes), not all of the text may be available. +If the event was generated by a large amount of text (such as +a start tag for an element with many attributes), not all of the +text may be available. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self) -/*[clinic end generated code: output=a88026d683fc22cc input=925cea010fdfa682]*/ +/*[clinic end generated code: output=a88026d683fc22cc input=a672f48f09bb73d2]*/ { if (self->in_callback) { int offset, size; @@ -1191,7 +1194,6 @@ pyexpat_xmlparser_SetParamEntityParsing_impl(xmlparseobject *self, int flag) #if XML_COMBINED_VERSION >= 19505 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.UseForeignDTD cls: defining_class @@ -1200,15 +1202,16 @@ pyexpat.xmlparser.UseForeignDTD Allows the application to provide an artificial external subset if one is not specified as part of the document instance. -This readily allows the use of a 'default' document type controlled by the -application, while still getting the advantage of providing document type -information to the parser. 'flag' defaults to True if not provided. +This readily allows the use of a 'default' document type controlled +by the application, while still getting the advantage of providing +document type information to the parser. 'flag' defaults to True if +not provided. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, int flag) -/*[clinic end generated code: output=d7d98252bd25a20f input=c2264845d8c0029c]*/ +/*[clinic end generated code: output=d7d98252bd25a20f input=2920baa5bf24714d]*/ { pyexpat_state *state = PyType_GetModuleState(cls); enum XML_Error rc; @@ -1268,7 +1271,6 @@ set_maximum_amplification(xmlparseobject *self, #if XML_COMBINED_VERSION >= 20400 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold cls: defining_class @@ -1277,21 +1279,22 @@ pyexpat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold Sets the number of output bytes needed to activate protection against billion laughs attacks. -The number of output bytes includes amplification from entity expansion -and reading DTD files. +The number of output bytes includes amplification from entity +expansion and reading DTD files. -Parser objects usually have a protection activation threshold of 8 MiB, -but the actual default value depends on the underlying Expat library. +Parser objects usually have a protection activation threshold of +8 MiB, but the actual default value depends on the underlying Expat +library. -Activation thresholds below 4 MiB are known to break support for DITA 1.3 -payload and are hence not recommended. +Activation thresholds below 4 MiB are known to break support for +DITA 1.3 payload and are hence not recommended. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlparseobject *self, PyTypeObject *cls, unsigned long long threshold) -/*[clinic end generated code: output=0c082342f1c78114 input=fa2f91f26b62a42a]*/ +/*[clinic end generated code: output=0c082342f1c78114 input=8d84b0e3a873cdba]*/ { return set_activation_threshold( self, cls, threshold, @@ -1303,7 +1306,6 @@ pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlpa #if XML_COMBINED_VERSION >= 20400 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification cls: defining_class @@ -1312,25 +1314,28 @@ pyexpat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification Sets the maximum tolerated amplification factor for protection against billion laughs attacks. -The amplification factor is calculated as "(direct + indirect) / direct" -while parsing, where "direct" is the number of bytes read from the primary -document in parsing and "indirect" is the number of bytes added by expanding -entities and reading external DTD files, combined. +The amplification factor is calculated as "(direct + indirect) / +direct" while parsing, where "direct" is the number of bytes read +from the primary document in parsing and "indirect" is the number of +bytes added by expanding entities and reading external DTD files, +combined. -The 'max_factor' value must be a non-NaN floating point value greater than -or equal to 1.0. Amplification factors greater than 30,000 can be observed -in the middle of parsing even with benign files in practice. In particular, -the activation threshold should be carefully chosen to avoid false positives. +The 'max_factor' value must be a non-NaN floating point value +greater than or equal to 1.0. Amplification factors greater than +30,000 can be observed in the middle of parsing even with benign +files in practice. In particular, the activation threshold should +be carefully chosen to avoid false positives. Parser objects usually have a maximum amplification factor of 100, -but the actual default value depends on the underlying Expat library. +but the actual default value depends on the underlying Expat +library. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlparseobject *self, PyTypeObject *cls, float max_factor) -/*[clinic end generated code: output=c590439eadf463fa input=cc1e97c1fd2bd950]*/ +/*[clinic end generated code: output=c590439eadf463fa input=d0f11971c5b9e98b]*/ { return set_maximum_amplification( self, cls, max_factor, @@ -1342,7 +1347,6 @@ pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlp #if XML_COMBINED_VERSION >= 20702 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.SetAllocTrackerActivationThreshold cls: defining_class @@ -1351,15 +1355,16 @@ pyexpat.xmlparser.SetAllocTrackerActivationThreshold Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM. -Parser objects usually have an allocation activation threshold of 64 MiB, -but the actual default value depends on the underlying Expat library. +Parser objects usually have an allocation activation threshold of +64 MiB, but the actual default value depends on the underlying Expat +library. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, PyTypeObject *cls, unsigned long long threshold) -/*[clinic end generated code: output=bed7e93207ba08c5 input=b7a7a3e3d054286a]*/ +/*[clinic end generated code: output=bed7e93207ba08c5 input=4728360b545de87a]*/ { return set_activation_threshold( self, cls, threshold, @@ -1371,7 +1376,6 @@ pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, #if XML_COMBINED_VERSION >= 20702 /*[clinic input] @permit_long_summary -@permit_long_docstring_body pyexpat.xmlparser.SetAllocTrackerMaximumAmplification cls: defining_class @@ -1380,25 +1384,27 @@ pyexpat.xmlparser.SetAllocTrackerMaximumAmplification Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated. -The amplification factor is calculated as "allocated / direct" while parsing, -where "direct" is the number of bytes read from the primary document in parsing -and "allocated" is the number of bytes of dynamic memory allocated in the parser -hierarchy. +The amplification factor is calculated as "allocated / direct" while +parsing, where "direct" is the number of bytes read from the primary +document in parsing and "allocated" is the number of bytes of +dynamic memory allocated in the parser hierarchy. -The 'max_factor' value must be a non-NaN floating point value greater than -or equal to 1.0. Amplification factors greater than 100.0 can be observed -near the start of parsing even with benign files in practice. In particular, -the activation threshold should be carefully chosen to avoid false positives. +The 'max_factor' value must be a non-NaN floating point value +greater than or equal to 1.0. Amplification factors greater than +100.0 can be observed near the start of parsing even with benign +files in practice. In particular, the activation threshold should +be carefully chosen to avoid false positives. Parser objects usually have a maximum amplification factor of 100, -but the actual default value depends on the underlying Expat library. +but the actual default value depends on the underlying Expat +library. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl(xmlparseobject *self, PyTypeObject *cls, float max_factor) -/*[clinic end generated code: output=6e44bd48c9b112a0 input=c6af7ccb76ae5c6b]*/ +/*[clinic end generated code: output=6e44bd48c9b112a0 input=dd23ea3ef2069b69]*/ { return set_maximum_amplification( self, cls, max_factor, @@ -1433,6 +1439,60 @@ static struct PyMethodDef xmlparse_methods[] = { Make it as simple as possible. */ +typedef struct { + int map[256]; + char name[0]; +} pyexpat_encoding_info; + +static pyexpat_encoding_info * +pyexpat_encoding_create(const char *name, PyObject *mapping) +{ + if (!PyTuple_Check(mapping) || PyTuple_GET_SIZE(mapping) != 256) { + PyErr_SetString(PyExc_ValueError, + "_expat_decoding_table must be a 256-tuple of integers"); + return NULL; + } + pyexpat_encoding_info *info = (pyexpat_encoding_info *)PyMem_Malloc( + sizeof(pyexpat_encoding_info) + strlen(name) + 1); + if (info == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (int i = 0; i < 256; i++) { + int j = PyLong_AsInt(PyTuple_GET_ITEM(mapping, i)); + if (j == -1 && PyErr_Occurred()) { + PyMem_Free(info); + return NULL; + } + info->map[i] = j; + } + strcpy(info->name, name); + return info; +} + +static int +pyexpat_encoding_convert(void *data, const char *s) +{ + if (PyErr_Occurred()) { + return -1; + } + pyexpat_encoding_info *info = (pyexpat_encoding_info *)data; + int i = (unsigned char)s[0]; + assert(info->map[i] < -1); + PyObject *u = PyUnicode_Decode(s, -info->map[i], info->name, NULL); + if (u == NULL) { + return -1; + } + if (PyUnicode_GET_LENGTH(u) != 1) { + Py_DECREF(u); + return -1; + } + Py_UCS4 ch = PyUnicode_ReadChar(u, 0); + Py_DECREF(u); + return (int)ch; +} + + static const unsigned char template_buffer[256] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, @@ -1465,6 +1525,43 @@ PyUnknownEncodingHandler(void *encodingHandlerData, if (PyErr_Occurred()) return XML_STATUS_ERROR; + PyObject *codec = _PyCodec_LookupTextEncoding(name, NULL); + if (codec == NULL) { + return XML_STATUS_ERROR; + } + if (!PyTuple_CheckExact(codec)) { + PyObject *attr; + if (PyObject_GetOptionalAttrString(codec, "_expat_decoding_table", &attr) < 0) { + Py_DECREF(codec); + return XML_STATUS_ERROR; + } + if (attr != NULL) { + if (attr == Py_False) { + Py_DECREF(attr); + Py_DECREF(codec); + PyErr_Format(PyExc_ValueError, + "encoding '%s' is not supported", + name); + return XML_STATUS_ERROR; + } + pyexpat_encoding_info *data = pyexpat_encoding_create(name, attr); + Py_DECREF(attr); + if (data == NULL) { + Py_DECREF(codec); + return XML_STATUS_ERROR; + } + for (i = 0; i < 256; i++) { + info->map[i] = data->map[i]; + } + info->data = data; + info->convert = pyexpat_encoding_convert; + info->release = PyMem_Free; + Py_DECREF(codec); + return XML_STATUS_OK; + } + } + Py_DECREF(codec); + u = PyUnicode_Decode((const char*) template_buffer, 256, name, "replace"); if (u == NULL) { Py_XDECREF(u); @@ -1473,8 +1570,9 @@ PyUnknownEncodingHandler(void *encodingHandlerData, if (PyUnicode_GET_LENGTH(u) != 256) { Py_DECREF(u); - PyErr_SetString(PyExc_ValueError, - "multi-byte encodings are not supported"); + PyErr_Format(PyExc_ValueError, + "multi-byte encoding '%s' is not supported", + name); return XML_STATUS_ERROR; } @@ -1533,7 +1631,10 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, Py_DECREF(self); return NULL; } -#if XML_COMBINED_VERSION >= 20100 +#if XML_COMBINED_VERSION >= 20800 + /* This feature was added upstream in libexpat 2.8.0. */ + XML_SetHashSalt16Bytes(self->itself, _Py_HashSecret.expat.hashsalt16); +#elif XML_COMBINED_VERSION >= 20100 /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.expat.hashsalt); @@ -2427,6 +2528,11 @@ pyexpat_exec(PyObject *mod) #else capi->SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20800 + capi->SetHashSalt16Bytes = XML_SetHashSalt16Bytes; +#else + capi->SetHashSalt16Bytes = NULL; +#endif #if XML_COMBINED_VERSION >= 20600 capi->SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; #else @@ -2443,8 +2549,8 @@ pyexpat_exec(PyObject *mod) capi->SetBillionLaughsAttackProtectionActivationThreshold = XML_SetBillionLaughsAttackProtectionActivationThreshold; capi->SetBillionLaughsAttackProtectionMaximumAmplification = XML_SetBillionLaughsAttackProtectionMaximumAmplification; #else - capi->SetAllocTrackerActivationThreshold = NULL; - capi->SetAllocTrackerMaximumAmplification = NULL; + capi->SetBillionLaughsAttackProtectionActivationThreshold = NULL; + capi->SetBillionLaughsAttackProtectionMaximumAmplification = NULL; #endif /* export using capsule */ diff --git a/Modules/readline.c b/Modules/readline.c index 488332f548e5fe3..c580d2022fccf3d 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -432,6 +432,7 @@ readline_append_history_file_impl(PyObject *module, int nelements, /* Set history length */ /*[clinic input] +@permit_long_summary readline.set_history_length length: int @@ -444,7 +445,7 @@ A negative length is used to inhibit history truncation. static PyObject * readline_set_history_length_impl(PyObject *module, int length) -/*[clinic end generated code: output=e161a53e45987dc7 input=b8901bf16488b760]*/ +/*[clinic end generated code: output=e161a53e45987dc7 input=8d02c81b38ef81ec]*/ { FT_ATOMIC_STORE_INT_RELAXED(_history_length, length); Py_RETURN_NONE; @@ -453,6 +454,7 @@ readline_set_history_length_impl(PyObject *module, int length) /* Get history length */ /*[clinic input] +@permit_long_summary readline.get_history_length Return the maximum number of lines that will be written to the history file. @@ -460,7 +462,7 @@ Return the maximum number of lines that will be written to the history file. static PyObject * readline_get_history_length_impl(PyObject *module) -/*[clinic end generated code: output=83a2eeae35b6d2b9 input=5dce2eeba4327817]*/ +/*[clinic end generated code: output=83a2eeae35b6d2b9 input=a65823e732ebfa9d]*/ { int history_length = FT_ATOMIC_LOAD_INT_RELAXED(_history_length); return PyLong_FromLong(history_length); @@ -1404,6 +1406,10 @@ setup_readline(readlinestate *mod_state) completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ + + if (!completer_word_break_characters) { + goto error; + } #ifdef WITH_EDITLINE // libedit uses rl_basic_word_break_characters instead of // rl_completer_word_break_characters as complete delimiter @@ -1447,6 +1453,10 @@ setup_readline(readlinestate *mod_state) RESTORE_LOCALE(saved_locale) return 0; + +error: + RESTORE_LOCALE(saved_locale) + return -1; } /* Wrapper around GNU readline that handles signals differently. */ diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index eb3148ef24631bb..2c56dbc6a541f7a 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -242,7 +242,6 @@ set2list(fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) #endif /* FD_SETSIZE > 1024 */ /*[clinic input] -@permit_long_docstring_body select.select rlist: object @@ -253,7 +252,8 @@ select.select Wait until one or more file descriptors are ready for some kind of I/O. -The first three arguments are iterables of file descriptors to be waited for: +The first three arguments are iterables of file descriptors to be waited +for: rlist -- wait until ready for reading wlist -- wait until ready for writing xlist -- wait for an "exceptional condition" @@ -266,9 +266,9 @@ The optional 4th argument specifies a timeout in seconds; it may be a non-integer to specify fractions of seconds. If it is absent or None, the call will never time out. -The return value is a tuple of three lists corresponding to the first three -arguments; each contains the subset of the corresponding file descriptors -that are ready. +The return value is a tuple of three lists corresponding to the first +three arguments; each contains the subset of the corresponding file +descriptors that are ready. *** IMPORTANT NOTICE *** On Windows, only sockets are supported; on Unix, all file @@ -278,7 +278,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=b0403de75cd11cc1]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=cc93e9bb9ffacbaf]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -616,13 +616,13 @@ select.poll.poll Polls the set of registered file descriptors. -Returns a list containing any descriptors that have events or errors to -report, as a list of (fd, event) 2-tuples. +Returns a list containing any descriptors that have events or errors +to report, as a list of (fd, event) 2-tuples. [clinic start generated code]*/ static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=54310631457efdec]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=e0a9c0aa283de8c8]*/ { PyObject *result_list = NULL; int poll_result, i, j; @@ -975,19 +975,19 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) @critical_section select.devpoll.poll timeout as timeout_obj: object = None - The maximum time to wait in milliseconds, or else None (or a negative - value) to wait indefinitely. + The maximum time to wait in milliseconds, or else None (or + a negative value) to wait indefinitely. / Polls the set of registered file descriptors. -Returns a list containing any descriptors that have events or errors to -report, as a list of (fd, event) 2-tuples. +Returns a list containing any descriptors that have events or errors +to report, as a list of (fd, event) 2-tuples. [clinic start generated code]*/ static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=fe7a3f6dcbc118c5]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=9e1672658d728539]*/ { struct dvpoll dvp; PyObject *result_list = NULL; @@ -1233,18 +1233,17 @@ static PyType_Spec devpoll_Type_spec = { /*[clinic input] -@permit_long_docstring_body select.poll Returns a polling object. -This object supports registering and unregistering file descriptors, and then -polling them for I/O events. +This object supports registering and unregistering file descriptors, and +then polling them for I/O events. [clinic start generated code]*/ static PyObject * select_poll_impl(PyObject *module) -/*[clinic end generated code: output=16a665a4e1d228c5 input=5e07eea8ad564e7f]*/ +/*[clinic end generated code: output=16a665a4e1d228c5 input=0aefd4527e99e0aa]*/ { return (PyObject *)newPollObject(module); } @@ -1252,18 +1251,17 @@ select_poll_impl(PyObject *module) #ifdef HAVE_SYS_DEVPOLL_H /*[clinic input] -@permit_long_docstring_body select.devpoll Returns a polling object. -This object supports registering and unregistering file descriptors, and then -polling them for I/O events. +This object supports registering and unregistering file descriptors, and +then polling them for I/O events. [clinic start generated code]*/ static PyObject * select_devpoll_impl(PyObject *module) -/*[clinic end generated code: output=ea9213cc87fd9581 input=048506faef19d947]*/ +/*[clinic end generated code: output=ea9213cc87fd9581 input=4c2ac27d10248526]*/ { return (PyObject *)newDevPollObject(module); } @@ -1540,6 +1538,7 @@ pyepoll_internal_ctl(int epfd, int op, int fd, unsigned int events) } /*[clinic input] +@permit_long_summary select.epoll.register fd: fildes @@ -1555,7 +1554,7 @@ The epoll interface supports all file descriptors that support poll. static PyObject * select_epoll_register_impl(pyEpoll_Object *self, int fd, unsigned int eventmask) -/*[clinic end generated code: output=318e5e6386520599 input=a5071b71edfe3578]*/ +/*[clinic end generated code: output=318e5e6386520599 input=9f0c9ebb25a4fc8f]*/ { return pyepoll_internal_ctl(self->epfd, EPOLL_CTL_ADD, fd, eventmask); } @@ -1606,14 +1605,14 @@ select.epoll.poll Wait for events on the epoll file descriptor. -Returns a list containing any descriptors that have events to report, -as a list of (fd, events) 2-tuples. +Returns a list containing any descriptors that have events to +report, as a list of (fd, events) 2-tuples. [clinic start generated code]*/ static PyObject * select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int maxevents) -/*[clinic end generated code: output=e02d121a20246c6c input=deafa7f04a60ebe0]*/ +/*[clinic end generated code: output=e02d121a20246c6c input=911ddc16978a9159]*/ { int nfds, i; PyObject *elist = NULL, *etuple = NULL; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index fb548b8ca00f24e..8456239dee202d3 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -450,7 +450,6 @@ signal_raise_signal_impl(PyObject *module, int signalnum) } /*[clinic input] -@permit_long_docstring_body signal.signal signalnum: int @@ -460,16 +459,17 @@ signal.signal Set the action for the given signal. The action can be SIG_DFL, SIG_IGN, or a callable Python object. -The previous action is returned. See getsignal() for possible return values. +The previous action is returned. See getsignal() for possible return +values. *** IMPORTANT NOTICE *** -A signal handler function is called with two arguments: -the first is the signal number, the second is the interrupted stack frame. +A signal handler function is called with two arguments: the first is +the signal number, the second is the interrupted stack frame. [clinic start generated code]*/ static PyObject * signal_signal_impl(PyObject *module, int signalnum, PyObject *handler) -/*[clinic end generated code: output=b44cfda43780f3a1 input=7608656f34fa378b]*/ +/*[clinic end generated code: output=b44cfda43780f3a1 input=99ce4035ec56ffc1]*/ { _signal_module_state *modstate = get_signal_state(module); PyObject *old_handler; @@ -839,7 +839,6 @@ PySignal_SetWakeupFd(int fd) #ifdef HAVE_SETITIMER /*[clinic input] -@permit_long_docstring_body signal.setitimer which: int @@ -849,8 +848,8 @@ signal.setitimer Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF). -The timer will fire after value seconds and after that every interval seconds. -The itimer can be cleared by setting seconds to zero. +The timer will fire after value seconds and after that every interval +seconds. The itimer can be cleared by setting seconds to zero. Returns old values as a tuple: (delay, interval). [clinic start generated code]*/ @@ -858,7 +857,7 @@ Returns old values as a tuple: (delay, interval). static PyObject * signal_setitimer_impl(PyObject *module, int which, PyObject *seconds, PyObject *interval) -/*[clinic end generated code: output=65f9dcbddc35527b input=ab5bf2b8f5cff3f4]*/ +/*[clinic end generated code: output=65f9dcbddc35527b input=bd9f0d2ed8614193]*/ { _signal_module_state *modstate = get_signal_state(module); @@ -1019,13 +1018,13 @@ signal.sigwait Wait for a signal. Suspend execution of the calling thread until the delivery of one of the -signals specified in the signal set sigset. The function accepts the signal -and returns the signal number. +signals specified in the signal set sigset. The function accepts the +signal and returns the signal number. [clinic start generated code]*/ static PyObject * signal_sigwait_impl(PyObject *module, sigset_t sigset) -/*[clinic end generated code: output=f43770699d682f96 input=a6fbd47b1086d119]*/ +/*[clinic end generated code: output=f43770699d682f96 input=91773742dd416a3e]*/ { int err, signum; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f5993fc8fdaab28..3e82af3194d053a 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -654,10 +654,6 @@ class _socket.socket "PySocketSockObject *" "clinic_state()->sock_type" [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=2db2489bd2219fd8]*/ -#define clinic_state() (find_module_state_by_def(type)) -#include "clinic/socketmodule.c.h" -#undef clinic_state - /* XXX There's a problem here: *static* functions are not supposed to have a Py prefix (or use CapitalizedWords). Later... */ @@ -688,6 +684,18 @@ class _socket.socket "PySocketSockObject *" "clinic_state()->sock_type" #define IS_SELECTABLE(s) (_PyIsSelectable_fd((s)->sock_fd) || (s)->sock_timeout <= 0) #endif +// SCM_RIGHTS, sendmsg(), recvmsg() and sethostname() don't work properly on +// Cygwin: disable these features. +#ifdef __CYGWIN__ +# undef CMSG_LEN +# undef SCM_RIGHTS +# undef HAVE_SETHOSTNAME +#endif + +#define clinic_state() (find_module_state_by_def(type)) +#include "clinic/socketmodule.c.h" +#undef clinic_state + static PyObject* select_error(void) { @@ -4518,17 +4526,19 @@ sock_recvmsg_into(PyObject *self, PyObject *args) struct iovec *iovs = NULL; Py_ssize_t i, nitems, nbufs = 0; Py_buffer *bufs = NULL; - PyObject *buffers_arg, *fast, *retval = NULL; + PyObject *buffers_arg, *buffers_tuple, *retval = NULL; if (!PyArg_ParseTuple(args, "O|ni:recvmsg_into", &buffers_arg, &ancbufsize, &flags)) return NULL; - if ((fast = PySequence_Fast(buffers_arg, - "recvmsg_into() argument 1 must be an " - "iterable")) == NULL) + buffers_tuple = PySequence_Tuple(buffers_arg); + if (buffers_tuple == NULL) { + PyErr_SetString(PyExc_TypeError, + "recvmsg_into() argument 1 must be an iterable"); return NULL; - nitems = PySequence_Fast_GET_SIZE(fast); + } + nitems = PyTuple_GET_SIZE(buffers_tuple); if (nitems > INT_MAX) { PyErr_SetString(PyExc_OSError, "recvmsg_into() argument 1 is too long"); goto finally; @@ -4542,7 +4552,7 @@ sock_recvmsg_into(PyObject *self, PyObject *args) goto finally; } for (; nbufs < nitems; nbufs++) { - if (!PyArg_Parse(PySequence_Fast_GET_ITEM(fast, nbufs), + if (!PyArg_Parse(PyTuple_GET_ITEM(buffers_tuple, nbufs), "w*;recvmsg_into() argument 1 must be an iterable " "of single-segment read-write buffers", &bufs[nbufs])) @@ -4558,7 +4568,7 @@ sock_recvmsg_into(PyObject *self, PyObject *args) PyBuffer_Release(&bufs[i]); PyMem_Free(bufs); PyMem_Free(iovs); - Py_DECREF(fast); + Py_DECREF(buffers_tuple); return retval; } @@ -4620,7 +4630,6 @@ sock_send_impl(PySocketSockObject *s, void *data) } /*[clinic input] -@permit_long_docstring_body _socket.socket.send self as s: self(type="PySocketSockObject *") data as pbuf: Py_buffer @@ -4630,12 +4639,13 @@ _socket.socket.send Send a data string to the socket. For the optional flags argument, see the Unix manual. -Return the number of bytes sent; this may be less than len(data) if the network is busy. +Return the number of bytes sent; this may be less than len(data) if +the network is busy. [clinic start generated code]*/ static PyObject * _socket_socket_send_impl(PySocketSockObject *s, Py_buffer *pbuf, int flags) -/*[clinic end generated code: output=3ddf83f17d0c875b input=e776a48af2e3d615]*/ +/*[clinic end generated code: output=3ddf83f17d0c875b input=d2b8af9bf99cfafb]*/ { struct sock_send ctx; @@ -4665,13 +4675,14 @@ Send a data string to the socket. For the optional flags argument, see the Unix manual. This calls send() repeatedly until all data is sent. -If an error occurs, it's impossible to tell how much data has been sent. +If an error occurs, it's impossible to tell how much data has been +sent. [clinic start generated code]*/ static PyObject * _socket_socket_sendall_impl(PySocketSockObject *s, Py_buffer *pbuf, int flags) -/*[clinic end generated code: output=ec92861424d3faa8 input=732b15b9ca64dce6]*/ +/*[clinic end generated code: output=ec92861424d3faa8 input=2600de13b4614893]*/ { char *buf; @@ -4852,14 +4863,14 @@ sock_sendmsg_iovec(PySocketSockObject *s, PyObject *data_arg, /* Fill in an iovec for each message part, and save the Py_buffer structs to release afterwards. */ - data_fast = PySequence_Fast(data_arg, - "sendmsg() argument 1 must be an " - "iterable"); + data_fast = PySequence_Tuple(data_arg); if (data_fast == NULL) { + PyErr_SetString(PyExc_TypeError, + "sendmsg() argument 1 must be an iterable"); goto finally; } - ndataparts = PySequence_Fast_GET_SIZE(data_fast); + ndataparts = PyTuple_GET_SIZE(data_fast); if (ndataparts > INT_MAX) { PyErr_SetString(PyExc_OSError, "sendmsg() argument 1 is too long"); goto finally; @@ -4881,7 +4892,7 @@ sock_sendmsg_iovec(PySocketSockObject *s, PyObject *data_arg, } } for (; ndatabufs < ndataparts; ndatabufs++) { - if (PyObject_GetBuffer(PySequence_Fast_GET_ITEM(data_fast, ndatabufs), + if (PyObject_GetBuffer(PyTuple_GET_ITEM(data_fast, ndatabufs), &databufs[ndatabufs], PyBUF_SIMPLE) < 0) goto finally; iovs[ndatabufs].iov_base = databufs[ndatabufs].buf; @@ -4921,20 +4932,20 @@ The buffers argument specifies the non-ancillary data as an iterable of bytes-like objects (e.g. bytes objects). The ancdata argument specifies the ancillary data (control messages) as an iterable of zero or more tuples (cmsg_level, cmsg_type, -cmsg_data), where cmsg_level and cmsg_type are integers specifying the -protocol level and protocol-specific type respectively, and cmsg_data -is a bytes-like object holding the associated data. The flags -argument defaults to 0 and has the same meaning as for send(). If -address is supplied and not None, it sets a destination address for -the message. The return value is the number of bytes of non-ancillary -data sent. +cmsg_data), where cmsg_level and cmsg_type are integers specifying +the protocol level and protocol-specific type respectively, and +cmsg_data is a bytes-like object holding the associated data. The +flags argument defaults to 0 and has the same meaning as for send(). +If address is supplied and not None, it sets a destination address +for the message. The return value is the number of bytes of +non-ancillary data sent. [clinic start generated code]*/ static PyObject * _socket_socket_sendmsg_impl(PySocketSockObject *s, PyObject *data_arg, PyObject *cmsg_arg, int flags, PyObject *addr_arg) -/*[clinic end generated code: output=3b4cb1110644ce39 input=479c13d90bd2f88b]*/ +/*[clinic end generated code: output=3b4cb1110644ce39 input=8ae408971a3aa329]*/ { Py_ssize_t i, ndatabufs = 0, ncmsgs, ncmsgbufs = 0; @@ -7310,6 +7321,7 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) /*[clinic input] +@permit_long_summary _socket.if_indextoname if_index as index: NET_IFINDEX / @@ -7319,7 +7331,7 @@ Returns the interface name corresponding to the interface index if_index. static PyObject * _socket_if_indextoname_impl(PyObject *module, NET_IFINDEX index) -/*[clinic end generated code: output=e48bc324993052e0 input=c93f753d0cf6d7d1]*/ +/*[clinic end generated code: output=e48bc324993052e0 input=2a0026b271cd43ae]*/ { errno = ENXIO; // in case 'if_indextoname' does not set errno char name[IF_NAMESIZE + 1]; @@ -9283,6 +9295,9 @@ socket_exec(PyObject *m) /* Initialize gethostbyname lock */ #if defined(USE_GETHOSTBYNAME_LOCK) netdb_lock = PyThread_allocate_lock(); + if (netdb_lock == NULL) { + goto error; + } #endif #ifdef MS_WINDOWS diff --git a/Modules/termios.c b/Modules/termios.c index 95b9c920f39c126..38743e176f0bf6c 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -312,6 +312,7 @@ termios_tcsendbreak_impl(PyObject *module, int fd, int duration) } /*[clinic input] +@permit_long_summary termios.tcdrain fd: fildes @@ -322,7 +323,7 @@ Wait until all output written to file descriptor fd has been transmitted. static PyObject * termios_tcdrain_impl(PyObject *module, int fd) -/*[clinic end generated code: output=5fd86944c6255955 input=c99241b140b32447]*/ +/*[clinic end generated code: output=5fd86944c6255955 input=d1557e60b5ec66c5]*/ { termiosmodulestate *state = PyModule_GetState(module); int r; @@ -474,7 +475,6 @@ termios_tcgetwinsize_impl(PyObject *module, int fd) } /*[clinic input] -@permit_long_docstring_body termios.tcsetwinsize fd: fildes @@ -484,12 +484,13 @@ termios.tcsetwinsize Set the tty winsize for file descriptor fd. The winsize to be set is taken from the winsize argument, which -is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize(). +is a two-item tuple (ws_row, ws_col) like the one returned by +tcgetwinsize(). [clinic start generated code]*/ static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz) -/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=9a163c4e06fc4a41]*/ +/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=efc9beb16d06382a]*/ { if (!PySequence_Check(winsz) || PySequence_Size(winsz) != 2) { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 25e744d7da25c72..d90bf1f2ef90ed9 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -820,12 +820,15 @@ time_strftime1(time_char **outbuf, size_t *bufsize, PyErr_NoMemory(); return NULL; } - *outbuf = (time_char *)PyMem_Realloc(*outbuf, - *bufsize*sizeof(time_char)); - if (*outbuf == NULL) { + time_char *tmp = (time_char *)PyMem_Realloc(*outbuf, + *bufsize*sizeof(time_char)); + if (tmp == NULL) { + PyMem_Free(*outbuf); + *outbuf = NULL; PyErr_NoMemory(); return NULL; } + *outbuf = tmp; #if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) errno = 0; #endif diff --git a/Modules/tkappinit.c b/Modules/tkappinit.c index 4c4081e43a8e3dd..1075ccf24d44877 100644 --- a/Modules/tkappinit.c +++ b/Modules/tkappinit.c @@ -37,7 +37,7 @@ Tcl_AppInit(Tcl_Interp *interp) return TCL_OK; } - if (Tk_Init(interp) == TCL_ERROR) { + if (Tkinter_TkInit(interp) == TCL_ERROR) { return TCL_ERROR; } diff --git a/Modules/tkinter.h b/Modules/tkinter.h index 40281c217603318..b73e99b28a40212 100644 --- a/Modules/tkinter.h +++ b/Modules/tkinter.h @@ -16,4 +16,6 @@ (TK_RELEASE_LEVEL << 8) | \ (TK_RELEASE_SERIAL << 0)) +int Tkinter_TkInit(Tcl_Interp *interp); + #endif /* !TKINTER_H */ diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 55b33a76e7af8a3..5f0e7ab6ec220d6 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -291,6 +291,7 @@ unicodedata_UCD_numeric_impl(PyObject *self, int chr, } /*[clinic input] +@permit_long_summary unicodedata.UCD.category self: self @@ -302,7 +303,7 @@ Returns the general category assigned to the character chr as string. static PyObject * unicodedata_UCD_category_impl(PyObject *self, int chr) -/*[clinic end generated code: output=8571539ee2e6783a input=27d6f3d85050bc06]*/ +/*[clinic end generated code: output=8571539ee2e6783a input=1d729c67299e8a31]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -316,6 +317,7 @@ unicodedata_UCD_category_impl(PyObject *self, int chr) } /*[clinic input] +@permit_long_summary unicodedata.UCD.bidirectional self: self @@ -329,7 +331,7 @@ If no such value is defined, an empty string is returned. static PyObject * unicodedata_UCD_bidirectional_impl(PyObject *self, int chr) -/*[clinic end generated code: output=d36310ce2039bb92 input=b3d8f42cebfcf475]*/ +/*[clinic end generated code: output=d36310ce2039bb92 input=838f8a2203bd2990]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -373,6 +375,7 @@ unicodedata_UCD_combining_impl(PyObject *self, int chr) } /*[clinic input] +@permit_long_summary unicodedata.UCD.mirrored -> int self: self @@ -387,7 +390,7 @@ character in bidirectional text, 0 otherwise. static int unicodedata_UCD_mirrored_impl(PyObject *self, int chr) -/*[clinic end generated code: output=2532dbf8121b50e6 input=5dd400d351ae6f3b]*/ +/*[clinic end generated code: output=2532dbf8121b50e6 input=6db28989e49cd9c8]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -403,6 +406,7 @@ unicodedata_UCD_mirrored_impl(PyObject *self, int chr) } /*[clinic input] +@permit_long_summary unicodedata.UCD.east_asian_width self: self @@ -414,7 +418,7 @@ Returns the east asian width assigned to the character chr as string. static PyObject * unicodedata_UCD_east_asian_width_impl(PyObject *self, int chr) -/*[clinic end generated code: output=484e8537d9ee8197 input=c4854798aab026e0]*/ +/*[clinic end generated code: output=484e8537d9ee8197 input=207c5f68fa475516]*/ { int index; Py_UCS4 c = (Py_UCS4)chr; @@ -552,19 +556,74 @@ get_decomp_record(PyObject *self, Py_UCS4 code, (*index)++; } +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(Py_UCS4 *data, Py_ssize_t length) +{ + for (Py_ssize_t i = 1; i < length; i++) { + Py_UCS4 code = data[i]; + unsigned char combining = _getrecord_ex(code)->combining; + Py_ssize_t j = i; + + while (j > 0) { + Py_UCS4 previous = data[j - 1]; + if (_getrecord_ex(previous)->combining <= combining) { + break; + } + data[j] = previous; + j--; + } + if (j != i) { + data[j] = code; + } + } +} + +static void +canonical_ordering_sort_counting(Py_UCS4 *data, Py_ssize_t length, + Py_UCS4 *sortbuf) +{ + Py_ssize_t counts[256] = {0}; + Py_ssize_t total = 0; + + for (Py_ssize_t i = 0; i < length; i++) { + Py_UCS4 code = data[i]; + unsigned char combining = _getrecord_ex(code)->combining; + counts[combining]++; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(counts); i++) { + Py_ssize_t count = counts[i]; + counts[i] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC. */ + for (Py_ssize_t i = 0; i < length; i++) { + Py_UCS4 code = data[i]; + unsigned char combining = _getrecord_ex(code)->combining; + sortbuf[counts[combining]++] = code; + } + memcpy(data, sortbuf, length * sizeof(Py_UCS4)); +} + static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { PyObject *result; Py_UCS4 *output; Py_ssize_t i, o, osize; - int kind; - const void *data; + int input_kind; + const void *input_data; /* Longest decomposition in Unicode 3.2: U+FDFA */ Py_UCS4 stack[20]; Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UCS4 *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_LENGTH(input); @@ -584,11 +643,11 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) return NULL; } i = o = 0; - kind = PyUnicode_KIND(input); - data = PyUnicode_DATA(input); + input_kind = PyUnicode_KIND(input); + input_data = PyUnicode_DATA(input); while (i < isize) { - stack[stackptr++] = PyUnicode_READ(kind, data, i++); + stack[stackptr++] = PyUnicode_READ(input_kind, input_data, i++); while(stackptr) { Py_UCS4 code = stack[--stackptr]; /* Hangul Decomposition adds three characters in @@ -650,55 +709,79 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) } } - result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - output, o); - PyMem_Free(output); - if (!result) - return NULL; - - kind = PyUnicode_KIND(result); - data = PyUnicode_DATA(result); - - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. */ i = 0; - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - for (i++; i < PyUnicode_GET_LENGTH(result); i++) { - cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < o) { + Py_ssize_t run_length, run_start; + int needs_sort = 0; + + Py_UCS4 ch = output[i]; + prev = _getrecord_ex(ch)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1); - PyUnicode_WRITE(kind, data, o+1, - PyUnicode_READ(kind, data, o)); - PyUnicode_WRITE(kind, data, o, tmp); - o--; - if (o < 0) - break; - prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < o) { + Py_UCS4 ch = output[i]; + cur = _getrecord_ex(ch)->combining; + if (cur == 0) { break; + } + if (prev > cur) { + needs_sort = 1; + } + prev = cur; + i++; + } + if (!needs_sort) { + continue; } - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(output + run_start, run_length); + continue; + } + + if (run_length > sortbuflen) { + Py_UCS4 *new_sortbuf = PyMem_Resize(sortbuf, Py_UCS4, run_length); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + PyMem_Free(output); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + + canonical_ordering_sort_counting(output + run_start, run_length, + sortbuf); } + PyMem_Free(sortbuf); + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, output, o); + PyMem_Free(output); return result; } static int find_nfc_index(const struct reindex* nfc, Py_UCS4 code) { - unsigned int index; - for (index = 0; nfc[index].start; index++) { - unsigned int start = nfc[index].start; - if (code < start) - return -1; - if (code <= start + nfc[index].count) { - unsigned int delta = code - start; - return nfc[index].index + delta; - } + /* The table is sorted by .start ascending with disjoint [start, start+count] + ranges and ends with a sentinel whose .start exceeds every codepoint, so + a single .start <= code test per entry also stops at the sentinel. Find + the first entry past code, then range-check the candidate (entry i - 1). */ + unsigned int i; + for (i = 0; (Py_UCS4)nfc[i].start <= code; i++) { + } + if (i == 0) { + return -1; + } + unsigned int start = nfc[i - 1].start; + if (code <= start + nfc[i - 1].count) { + return nfc[i - 1].index + (code - start); } return -1; } @@ -911,6 +994,7 @@ is_normalized_quickcheck(PyObject *self, PyObject *input, bool nfc, bool k, } /*[clinic input] +@permit_long_summary unicodedata.UCD.is_normalized self: self @@ -926,7 +1010,7 @@ Valid values for form are 'NFC', 'NFKC', 'NFD', and 'NFKD'. static PyObject * unicodedata_UCD_is_normalized_impl(PyObject *self, PyObject *form, PyObject *input) -/*[clinic end generated code: output=11e5a3694e723ca5 input=a544f14cea79e508]*/ +/*[clinic end generated code: output=11e5a3694e723ca5 input=de66aa679265300b]*/ { if (PyUnicode_GET_LENGTH(input) == 0) { /* special case empty input strings. */ @@ -1543,32 +1627,17 @@ capi_getcode(const char* name, int namelen, Py_UCS4* code, return _check_alias_and_seq(code, with_named_seq); } -static void -unicodedata_destroy_capi(PyObject *capsule) -{ - void *capi = PyCapsule_GetPointer(capsule, PyUnicodeData_CAPSULE_NAME); - PyMem_Free(capi); -} - static PyObject * unicodedata_create_capi(void) { - _PyUnicode_Name_CAPI *capi = PyMem_Malloc(sizeof(_PyUnicode_Name_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->getname = capi_getucname; - capi->getcode = capi_getcode; - - PyObject *capsule = PyCapsule_New(capi, - PyUnicodeData_CAPSULE_NAME, - unicodedata_destroy_capi); - if (capsule == NULL) { - PyMem_Free(capi); - } - return capsule; -}; + // Statically allocated so that any cached pointers stay valid after unicodedata + // is removed from sys.modules and the capsule is gc'd (gh-149449). + static _PyUnicode_Name_CAPI capi = { + .getname = capi_getucname, + .getcode = capi_getcode, + }; + return PyCapsule_New(&capi, PyUnicodeData_CAPSULE_NAME, NULL); +} /* -------------------------------------------------------------------- */ diff --git a/Modules/unicodedata_db.h b/Modules/unicodedata_db.h index 9e88f5cca7115b0..1c961c5e329341c 100644 --- a/Modules/unicodedata_db.h +++ b/Modules/unicodedata_db.h @@ -629,7 +629,7 @@ static struct reindex nfc_first[] = { { 93539, 0, 388}, { 93543, 0, 389}, { 93545, 0, 390}, - {0,0,0} + {0x7fffffff, 0, 0} }; static struct reindex nfc_last[] = { @@ -680,7 +680,7 @@ static struct reindex nfc_last[] = { { 90398, 2, 67}, { 90409, 0, 70}, { 93543, 0, 71}, - {0,0,0} + {0x7fffffff, 0, 0} }; /* string literals */ diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 09c8d9487f54266..96454ee5e83eab7 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -11,7 +11,13 @@ other files, you'll have to create a file "foobarobject.h"; see floatobject.h for an example. - This module roughly corresponds to:: + This module uses Limited API 3.15. + See ``xxlimited_3_13.c`` if you want to support older CPython versions. + + This module roughly corresponds to the following. + (All underscore-prefixed attributes are not accessible from Python.) + + :: class Xxo: """A class that explicitly stores attributes in an internal dict @@ -27,6 +33,8 @@ return self._x_attr[name] def __setattr__(self, name, value): + if name == "reserved": + raise AttributeError("cannot set 'reserved'") self._x_attr[name] = value def __delattr__(self, name): @@ -64,11 +72,13 @@ pass */ -// Need limited C API version 3.13 for Py_mod_gil -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 -#endif +// Target both flavors of the Stable ABI. +// Both are set to version 3.15, which adds PyModExport +// (When using a build tool, check if it has an option to set these +// so they do not need to be defined in the source.) +#define Py_LIMITED_API 0x030f0000 // abi3 (GIL-enabled builds) +#define Py_TARGET_ABI3T 0x030f0000 // abi3t (free-threaded builds) + #include "Python.h" #include <string.h> @@ -77,43 +87,135 @@ // Module state typedef struct { - PyObject *Xxo_Type; // Xxo class + PyTypeObject *Xxo_Type; // Xxo class PyObject *Error_Type; // Error class } xx_state; -/* Xxo objects */ +/* Xxo objects. + * + * A non-trivial extension type, intentionally showing a number of features + * that aren't easy to implement in the Limited API. + */ + +// Forward declaration +static PyType_Spec Xxo_Type_spec; + +// Get the module state (xx_state*) from a given type object 'type', which +// must be a subclass of Xxo (the type we're defining). +// This is complicated by the fact that the Xxo type is dynamically allocated, +// and there may be several such types in a given Python process -- for +// example, in different subinterpreters, or through loading this +// extension module several times. +// So, we don't have a "global" pointer to the type, or to the module, etc.; +// instead we search based on `Xxo_Type_spec` (which is static, immutable, +// and process-global). +// +// When possible, it's better to avoid `PyType_GetBaseByToken` -- for an +// example, see the `demo` method (Xxo_demo C function), which uses a +// "defining class". But, in many cases it's the best solution. +static xx_state * +Xxo_state_from_type(PyTypeObject *type) +{ + PyTypeObject *base; + // Search all superclasses of 'type' for one that was defined using + // "Xxo_Type_spec". That must be our 'Xxo' class. + if (PyType_GetBaseByToken(type, &Xxo_Type_spec, &base) < 0) { + return NULL; + } + if (base == NULL) { + PyErr_SetString(PyExc_TypeError, "need Xxo subclass"); + return NULL; + } + // From this type, get the associated module. That must be the + // relevant `xxlimited` module. + xx_state *state = PyType_GetModuleState(base); + Py_DECREF(base); + return state; +} -// Instance state +// Structure for data needed by the XxoObject type. +// Since the object may be shared across threads, access to the fields +// usually needs to be synchronized (using Py_BEGIN_CRITICAL_SECTION). typedef struct { - PyObject_HEAD - PyObject *x_attr; /* Attributes dictionary. - * May be NULL, which acts as an - * empty dict. - */ - char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ - Py_ssize_t x_exports; /* how many buffer are exported */ -} XxoObject; - -#define XxoObject_CAST(op) ((XxoObject *)(op)) -// TODO: full support for type-checking was added in 3.14 (Py_tp_token) -// #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) - -static XxoObject * -newXxoObject(PyObject *module) + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ + Py_ssize_t x_exports; /* how many buffers are exported */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer (for simplicity, + * this is constant, so does not need + * synchronization) + */ +} XxoObject_Data; + +// Get the `XxoObject_Data` structure for a given instance of our type. +static XxoObject_Data * +Xxo_get_data(PyObject *self) { - xx_state *state = PyModule_GetState(module); + xx_state *state = Xxo_state_from_type(Py_TYPE(self)); + if (!state) { + return NULL; + } + XxoObject_Data *data = PyObject_GetTypeData(self, state->Xxo_Type); + return data; +} + +// A variant of Xxo_get_data to be used in the tp_traverse handler. +// This function cannot have side effects (including reference count +// manipulation, creating objects, and raising exceptions), and must not +// call API functions that might have side effects. +// See: https://docs.python.org/3.15/c-api/gcsupport.html#traversal +static XxoObject_Data * +Xxo_get_data_DuringGC(PyObject *self) +{ + PyTypeObject *base; + PyType_GetBaseByToken_DuringGC(Py_TYPE(self), &Xxo_Type_spec, &base); + if (base == NULL) { + return NULL; + } + xx_state *state = PyType_GetModuleState_DuringGC(base); if (state == NULL) { return NULL; } - XxoObject *self; - self = PyObject_GC_New(XxoObject, (PyTypeObject*)state->Xxo_Type); + XxoObject_Data *data = PyObject_GetTypeData_DuringGC(self, state->Xxo_Type); + return data; +} + +// Xxo initialization +// This is the implementation of Xxo.__new__ +static PyObject * +Xxo_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + // Validate that we did not get any arguments. + if ((args != NULL && PyObject_Length(args)) + || (kwargs != NULL && PyObject_Length(kwargs))) + { + PyErr_SetString(PyExc_TypeError, "Xxo.__new__() takes no arguments"); + return NULL; + } + // Create an instance of *type* (which may be a subclass) + allocfunc alloc = PyType_GetSlot(type, Py_tp_alloc); + PyObject *self = alloc(type, 0); if (self == NULL) { return NULL; } - self->x_attr = NULL; - memset(self->x_buffer, 0, BUFSIZE); - self->x_exports = 0; + + // Initialize the C members on the instance. + // This is only included for the sake of example. The default alloc + // function zeroes instance memory; we don't need to do it again. + // Note that we during initialization (and finalization), we hold the only + // reference to the object, so we don't need to synchronize with + // other threads. + XxoObject_Data *xxo_data = Xxo_get_data(self); + if (xxo_data == NULL) { + Py_DECREF(self); + return NULL; + } + + xxo_data->x_attr = NULL; + memset(xxo_data->x_buffer, 0, BUFSIZE); + xxo_data->x_exports = 0; return self; } @@ -125,45 +227,63 @@ newXxoObject(PyObject *module) // traverse: Visit all references from an object, including its type static int -Xxo_traverse(PyObject *op, visitproc visit, void *arg) +Xxo_traverse(PyObject *self, visitproc visit, void *arg) { // Visit the type - Py_VISIT(Py_TYPE(op)); + Py_VISIT(Py_TYPE(self)); // Visit the attribute dict - XxoObject *self = XxoObject_CAST(op); - Py_VISIT(self->x_attr); + XxoObject_Data *data = Xxo_get_data_DuringGC(self); + if (data == NULL) { + return 0; + } + Py_VISIT(data->x_attr); return 0; } // clear: drop references in order to break all reference cycles static int -Xxo_clear(PyObject *op) +Xxo_clear(PyObject *self) { - XxoObject *self = XxoObject_CAST(op); - Py_CLEAR(self->x_attr); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return 0; + } + Py_CLEAR(data->x_attr); return 0; } // finalize: like clear, but should leave the object in a consistent state. // Equivalent to `__del__` in Python. static void -Xxo_finalize(PyObject *op) +Xxo_finalize(PyObject *self) { - XxoObject *self = XxoObject_CAST(op); - Py_CLEAR(self->x_attr); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return; + } + Py_CLEAR(data->x_attr); } // dealloc: drop all remaining references and free memory static void Xxo_dealloc(PyObject *self) { + // This function must preserve currently raised exception, if any. + PyObject *exc = PyErr_GetRaisedException(); + PyObject_GC_UnTrack(self); Xxo_finalize(self); + PyTypeObject *tp = Py_TYPE(self); freefunc free = PyType_GetSlot(tp, Py_tp_free); free(self); Py_DECREF(tp); + + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(NULL); + } + PyErr_SetRaisedException(exc); } @@ -171,11 +291,20 @@ Xxo_dealloc(PyObject *self) // Get an attribute. static PyObject * -Xxo_getattro(PyObject *op, PyObject *name) +Xxo_getattro(PyObject *self, PyObject *name) { - XxoObject *self = XxoObject_CAST(op); - if (self->x_attr != NULL) { - PyObject *v = PyDict_GetItemWithError(self->x_attr, name); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return 0; + } + + PyObject *x_attr; + Py_BEGIN_CRITICAL_SECTION(self); + x_attr = data->x_attr; + Py_END_CRITICAL_SECTION(); + + if (x_attr != NULL) { + PyObject *v = PyDict_GetItemWithError(x_attr, name); if (v != NULL) { return Py_NewRef(v); } @@ -185,24 +314,42 @@ Xxo_getattro(PyObject *op, PyObject *name) } // Fall back to generic implementation (this handles special attributes, // raising AttributeError, etc.) - return PyObject_GenericGetAttr(op, name); + return PyObject_GenericGetAttr(self, name); } // Set or delete an attribute. static int -Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) +Xxo_setattro(PyObject *self, PyObject *name, PyObject *v) { - XxoObject *self = XxoObject_CAST(op); - if (self->x_attr == NULL) { + // filter a specific attribute name + if (PyUnicode_Check(name) && PyUnicode_EqualToUTF8(name, "reserved")) { + PyErr_Format(PyExc_AttributeError, "cannot set %R", name); + return -1; + } + + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return -1; + } + + // If the attribute dict is not created yet, make one. + // This needs to be protected by a critical section to avoid another thread + // creating a duplicate dict. + PyObject *x_attr; + Py_BEGIN_CRITICAL_SECTION(self); + x_attr = data->x_attr; + if (x_attr == NULL) { // prepare the attribute dict - self->x_attr = PyDict_New(); - if (self->x_attr == NULL) { - return -1; - } + data->x_attr = x_attr = PyDict_New(); } + Py_END_CRITICAL_SECTION(); + if (x_attr == NULL) { + return -1; + } + if (v == NULL) { // delete an attribute - int rv = PyDict_DelItem(self->x_attr, name); + int rv = PyDict_DelItem(x_attr, name); if (rv < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { PyErr_SetString(PyExc_AttributeError, "delete non-existing Xxo attribute"); @@ -212,7 +359,7 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) } else { // set an attribute - return PyDict_SetItem(self->x_attr, name, v); + return PyDict_SetItem(x_attr, name, v); } } @@ -221,7 +368,7 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) */ static PyObject * -Xxo_demo(PyObject *op, PyTypeObject *defining_class, +Xxo_demo(PyObject *self, PyTypeObject *defining_class, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { if (kwnames != NULL && PyObject_Length(kwnames)) { @@ -260,30 +407,49 @@ static PyMethodDef Xxo_methods[] = { */ static int -Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) +Xxo_getbuffer(PyObject *self, Py_buffer *view, int flags) { - XxoObject *self = XxoObject_CAST(op); - int res = PyBuffer_FillInfo(view, op, - (void *)self->x_buffer, BUFSIZE, + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return -1; + } + int res = PyBuffer_FillInfo(view, self, + (void *)data->x_buffer, BUFSIZE, 0, flags); if (res == 0) { - self->x_exports++; + Py_BEGIN_CRITICAL_SECTION(self); + data->x_exports++; + Py_END_CRITICAL_SECTION(); } return res; } static void -Xxo_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) +Xxo_releasebuffer(PyObject *self, Py_buffer *Py_UNUSED(view)) { - XxoObject *self = XxoObject_CAST(op); - self->x_exports--; + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return; + } + Py_BEGIN_CRITICAL_SECTION(self); + data->x_exports--; + Py_END_CRITICAL_SECTION(); } static PyObject * -Xxo_get_x_exports(PyObject *op, void *Py_UNUSED(closure)) +Xxo_get_x_exports(PyObject *self, void *Py_UNUSED(closure)) { - XxoObject *self = XxoObject_CAST(op); - return PyLong_FromSsize_t(self->x_exports); + XxoObject_Data *data = Xxo_get_data(self); + if (data == NULL) { + return NULL; + } + Py_ssize_t result; + + Py_BEGIN_CRITICAL_SECTION(self); + result = data->x_exports; + Py_END_CRITICAL_SECTION(); + + return PyLong_FromSsize_t(result); } /* Xxo type definition */ @@ -299,6 +465,7 @@ static PyGetSetDef Xxo_getsetlist[] = { static PyType_Slot Xxo_Type_slots[] = { {Py_tp_doc, (char *)Xxo_doc}, + {Py_tp_new, Xxo_new}, {Py_tp_traverse, Xxo_traverse}, {Py_tp_clear, Xxo_clear}, {Py_tp_finalize, Xxo_finalize}, @@ -309,13 +476,14 @@ static PyType_Slot Xxo_Type_slots[] = { {Py_bf_getbuffer, Xxo_getbuffer}, {Py_bf_releasebuffer, Xxo_releasebuffer}, {Py_tp_getset, Xxo_getsetlist}, + {Py_tp_token, Py_TP_USE_SPEC}, {0, 0}, /* sentinel */ }; static PyType_Spec Xxo_Type_spec = { .name = "xxlimited.Xxo", - .basicsize = sizeof(XxoObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .basicsize = -(Py_ssize_t)sizeof(XxoObject_Data), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, .slots = Xxo_Type_slots, }; @@ -354,17 +522,17 @@ xx_foo(PyObject *module, PyObject *args) } -/* Function of no arguments returning new Xxo object */ +/* Function of no arguments returning new Xxo object. + * Note that a function exposed to Python with METH_NOARGS requires an unused + * second argument, so we cannot use newXxoObject directly. + */ static PyObject * xx_new(PyObject *module, PyObject *Py_UNUSED(unused)) { - XxoObject *rv; + xx_state *state = PyModule_GetState(module); - rv = newXxoObject(module); - if (rv == NULL) - return NULL; - return (PyObject *)rv; + return Xxo_new(state->Xxo_Type, NULL, NULL); } @@ -398,11 +566,12 @@ xx_modexec(PyObject *m) return -1; } - state->Xxo_Type = PyType_FromModuleAndSpec(m, &Xxo_Type_spec, NULL); + state->Xxo_Type = (PyTypeObject*)PyType_FromModuleAndSpec( + m, &Xxo_Type_spec, NULL); if (state->Xxo_Type == NULL) { return -1; } - if (PyModule_AddType(m, (PyTypeObject*)state->Xxo_Type) < 0) { + if (PyModule_AddType(m, state->Xxo_Type) < 0) { return -1; } @@ -410,12 +579,13 @@ xx_modexec(PyObject *m) // added to the module dict. // It does not inherit from "object" (PyObject_Type), but from "str" // (PyUnincode_Type). - PyObject *Str_Type = PyType_FromModuleAndSpec( + PyTypeObject *Str_Type = (PyTypeObject*)PyType_FromModuleAndSpec( m, &Str_Type_spec, (PyObject *)&PyUnicode_Type); if (Str_Type == NULL) { return -1; } - if (PyModule_AddType(m, (PyTypeObject*)Str_Type) < 0) { + if (PyModule_AddType(m, Str_Type) < 0) { + Py_DECREF(Str_Type); return -1; } Py_DECREF(Str_Type); @@ -423,29 +593,6 @@ xx_modexec(PyObject *m) return 0; } -static PyModuleDef_Slot xx_slots[] = { - - /* exec function to initialize the module (called as part of import - * after the object was added to sys.modules) - */ - {Py_mod_exec, xx_modexec}, - - /* Signal that this module supports being loaded in multiple interpreters - * with separate GILs (global interpreter locks). - * See "Isolating Extension Modules" on how to prepare a module for this: - * https://docs.python.org/3/howto/isolating-extensions.html - */ - {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, - - /* Signal that this module does not rely on the GIL for its own needs. - * Without this slot, free-threaded builds of CPython will enable - * the GIL when this module is loaded. - */ - {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - - {0, NULL} -}; - // Module finalization: modules that hold references in their module state // need to implement the fullowing GC hooks. They're similar to the ones for // types (see "Xxo finalization"). @@ -453,7 +600,10 @@ static PyModuleDef_Slot xx_slots[] = { static int xx_traverse(PyObject *module, visitproc visit, void *arg) { - xx_state *state = PyModule_GetState(module); + xx_state *state = PyModule_GetState_DuringGC(module); + if (state == NULL) { + return 0; + } Py_VISIT(state->Xxo_Type); Py_VISIT(state->Error_Type); return 0; @@ -463,6 +613,9 @@ static int xx_clear(PyObject *module) { xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return 0; + } Py_CLEAR(state->Xxo_Type); Py_CLEAR(state->Error_Type); return 0; @@ -473,27 +626,59 @@ xx_free(void *module) { // allow xx_modexec to omit calling xx_clear on error (void)xx_clear((PyObject *)module); + + xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return; + } } -static struct PyModuleDef xxmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "xxlimited", - .m_doc = module_doc, - .m_size = sizeof(xx_state), - .m_methods = xx_methods, - .m_slots = xx_slots, - .m_traverse = xx_traverse, - .m_clear = xx_clear, - .m_free = xx_free, +// Information that CPython uses to prevent loading incompatible extenstions +PyABIInfo_VAR(abi_info); + +static PySlot xx_slots[] = { + /* Basic metadata */ + PySlot_STATIC_DATA(Py_mod_name, "xxlimited"), + PySlot_STATIC_DATA(Py_mod_doc, (void*)module_doc), + PySlot_DATA(Py_mod_abi, &abi_info), + + /* The method table */ + PySlot_STATIC_DATA(Py_mod_methods, xx_methods), + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ + PySlot_FUNC(Py_mod_exec, xx_modexec), + + /* Module state and associated functions */ + PySlot_SIZE(Py_mod_state_size, sizeof(xx_state)), + PySlot_FUNC(Py_mod_state_traverse, xx_traverse), + PySlot_FUNC(Py_mod_state_clear, xx_clear), + PySlot_FUNC(Py_mod_state_free, xx_free), + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ + PySlot_DATA(Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED), + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + + PySlot_END }; -/* Export function for the module. *Must* be called PyInit_xx; usually it is - * the only non-`static` object in a module definition. +/* Export function for the module. *Must* be called PyModExport_xx; usually + * it is the only non-`static` object in a module definition. */ -PyMODINIT_FUNC -PyInit_xxlimited(void) +PyMODEXPORT_FUNC +PyModExport_xxlimited(void) { - return PyModuleDef_Init(&xxmodule); + return xx_slots; } diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index b0a682ac4e6bb69..9ef0eac9a924e6c 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -305,7 +305,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, #ifdef Py_GIL_DISABLED - // These definitions are in the limited API, but not until 3.13. + // In a free-threaded build, we don't use Limited API. {Py_mod_gil, Py_MOD_GIL_NOT_USED}, #endif {0, NULL} diff --git a/Modules/xxlimited_3_13.c b/Modules/xxlimited_3_13.c new file mode 100644 index 000000000000000..4f100f9150fc2a3 --- /dev/null +++ b/Modules/xxlimited_3_13.c @@ -0,0 +1,499 @@ +/* Use this file as a template to start implementing a module that + also declares object types. All occurrences of 'Xxo' should be changed + to something reasonable for your objects. After that, all other + occurrences of 'xx' should be changed to something reasonable for your + module. If your module is named foo your source file should be named + foo.c or foomodule.c. + + You will probably want to delete all references to 'x_attr' and add + your own types of attributes instead. Maybe you want to name your + local variables other than 'self'. If your object type is needed in + other files, you'll have to create a file "foobarobject.h"; see + floatobject.h for an example. + + This module roughly corresponds to:: + + class Xxo: + """A class that explicitly stores attributes in an internal dict + (to simulate custom attribute handling). + """ + + def __init__(self): + # In the C class, "_x_attr" is not accessible from Python code + self._x_attr = {} + self._x_exports = 0 + + def __getattr__(self, name): + return self._x_attr[name] + + def __setattr__(self, name, value): + self._x_attr[name] = value + + def __delattr__(self, name): + del self._x_attr[name] + + @property + def x_exports(self): + """Return the number of times an internal buffer is exported.""" + # Each Xxo instance has a 10-byte buffer that can be + # accessed via the buffer interface (e.g. `memoryview`). + return self._x_exports + + def demo(o, /): + if isinstance(o, str): + return o + elif isinstance(o, Xxo): + return o + else: + raise Error('argument must be str or Xxo') + + class Error(Exception): + """Exception raised by the xxlimited module""" + + def foo(i: int, j: int, /): + """Return the sum of i and j.""" + # Unlike this pseudocode, the C function will *only* work with + # integers and perform C long int arithmetic + return i + j + + def new(): + return Xxo() + + def Str(str): + # A trivial subclass of a built-in type + pass + */ + +// Need limited C API version 3.13 for Py_mod_gil +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030d0000 +#endif + +#include "Python.h" +#include <string.h> + +#define BUFSIZE 10 + +// Module state +typedef struct { + PyObject *Xxo_Type; // Xxo class + PyObject *Error_Type; // Error class +} xx_state; + + +/* Xxo objects */ + +// Instance state +typedef struct { + PyObject_HEAD + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ + char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ + Py_ssize_t x_exports; /* how many buffer are exported */ +} XxoObject; + +#define XxoObject_CAST(op) ((XxoObject *)(op)) +// TODO: full support for type-checking was added in 3.14 (Py_tp_token) +// #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) + +static XxoObject * +newXxoObject(PyObject *module) +{ + xx_state *state = PyModule_GetState(module); + if (state == NULL) { + return NULL; + } + XxoObject *self; + self = PyObject_GC_New(XxoObject, (PyTypeObject*)state->Xxo_Type); + if (self == NULL) { + return NULL; + } + self->x_attr = NULL; + memset(self->x_buffer, 0, BUFSIZE); + self->x_exports = 0; + return self; +} + +/* Xxo finalization. + * + * Types that store references to other PyObjects generally need to implement + * the GC slots: traverse, clear, dealloc, and (optionally) finalize. + */ + +// traverse: Visit all references from an object, including its type +static int +Xxo_traverse(PyObject *op, visitproc visit, void *arg) +{ + // Visit the type + Py_VISIT(Py_TYPE(op)); + + // Visit the attribute dict + XxoObject *self = XxoObject_CAST(op); + Py_VISIT(self->x_attr); + return 0; +} + +// clear: drop references in order to break all reference cycles +static int +Xxo_clear(PyObject *op) +{ + XxoObject *self = XxoObject_CAST(op); + Py_CLEAR(self->x_attr); + return 0; +} + +// finalize: like clear, but should leave the object in a consistent state. +// Equivalent to `__del__` in Python. +static void +Xxo_finalize(PyObject *op) +{ + XxoObject *self = XxoObject_CAST(op); + Py_CLEAR(self->x_attr); +} + +// dealloc: drop all remaining references and free memory +static void +Xxo_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Xxo_finalize(self); + PyTypeObject *tp = Py_TYPE(self); + freefunc free = PyType_GetSlot(tp, Py_tp_free); + free(self); + Py_DECREF(tp); +} + + +/* Xxo attribute handling */ + +// Get an attribute. +static PyObject * +Xxo_getattro(PyObject *op, PyObject *name) +{ + XxoObject *self = XxoObject_CAST(op); + if (self->x_attr != NULL) { + PyObject *v = PyDict_GetItemWithError(self->x_attr, name); + if (v != NULL) { + return Py_NewRef(v); + } + else if (PyErr_Occurred()) { + return NULL; + } + } + // Fall back to generic implementation (this handles special attributes, + // raising AttributeError, etc.) + return PyObject_GenericGetAttr(op, name); +} + +// Set or delete an attribute. +static int +Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) +{ + XxoObject *self = XxoObject_CAST(op); + if (self->x_attr == NULL) { + // prepare the attribute dict + self->x_attr = PyDict_New(); + if (self->x_attr == NULL) { + return -1; + } + } + if (v == NULL) { + // delete an attribute + int rv = PyDict_DelItem(self->x_attr, name); + if (rv < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_SetString(PyExc_AttributeError, + "delete non-existing Xxo attribute"); + return -1; + } + return rv; + } + else { + // set an attribute + return PyDict_SetItem(self->x_attr, name, v); + } +} + +/* Xxo methods: C functions plus a PyMethodDef array that lists them and + * specifies metadata. + */ + +static PyObject * +Xxo_demo(PyObject *op, PyTypeObject *defining_class, + PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + if (kwnames != NULL && PyObject_Length(kwnames)) { + PyErr_SetString(PyExc_TypeError, "demo() takes no keyword arguments"); + return NULL; + } + if (nargs != 1) { + PyErr_SetString(PyExc_TypeError, "demo() takes exactly 1 argument"); + return NULL; + } + + PyObject *o = args[0]; + + /* Test if the argument is "str" */ + if (PyUnicode_Check(o)) { + return Py_NewRef(o); + } + + /* test if the argument is of the Xxo class */ + if (PyObject_TypeCheck(o, defining_class)) { + return Py_NewRef(o); + } + + return Py_NewRef(Py_None); +} + +static PyMethodDef Xxo_methods[] = { + {"demo", _PyCFunction_CAST(Xxo_demo), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS, PyDoc_STR("demo(o) -> o")}, + {NULL, NULL} /* sentinel */ +}; + +/* Xxo buffer interface: C functions later referenced from PyType_Slot array. + * Other interfaces (e.g. for sequence-like or number-like types) are defined + * similarly. + */ + +static int +Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) +{ + XxoObject *self = XxoObject_CAST(op); + int res = PyBuffer_FillInfo(view, op, + (void *)self->x_buffer, BUFSIZE, + 0, flags); + if (res == 0) { + self->x_exports++; + } + return res; +} + +static void +Xxo_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) +{ + XxoObject *self = XxoObject_CAST(op); + self->x_exports--; +} + +static PyObject * +Xxo_get_x_exports(PyObject *op, void *Py_UNUSED(closure)) +{ + XxoObject *self = XxoObject_CAST(op); + return PyLong_FromSsize_t(self->x_exports); +} + +/* Xxo type definition */ + +PyDoc_STRVAR(Xxo_doc, + "A class that explicitly stores attributes in an internal dict"); + +static PyGetSetDef Xxo_getsetlist[] = { + {"x_exports", Xxo_get_x_exports, NULL, NULL}, + {NULL}, +}; + + +static PyType_Slot Xxo_Type_slots[] = { + {Py_tp_doc, (char *)Xxo_doc}, + {Py_tp_traverse, Xxo_traverse}, + {Py_tp_clear, Xxo_clear}, + {Py_tp_finalize, Xxo_finalize}, + {Py_tp_dealloc, Xxo_dealloc}, + {Py_tp_getattro, Xxo_getattro}, + {Py_tp_setattro, Xxo_setattro}, + {Py_tp_methods, Xxo_methods}, + {Py_bf_getbuffer, Xxo_getbuffer}, + {Py_bf_releasebuffer, Xxo_releasebuffer}, + {Py_tp_getset, Xxo_getsetlist}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Xxo_Type_spec = { + .name = "xxlimited_3_13.Xxo", + .basicsize = sizeof(XxoObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .slots = Xxo_Type_slots, +}; + + +/* Str type definition*/ + +static PyType_Slot Str_Type_slots[] = { + // slots array intentionally kept empty + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Str_Type_spec = { + .name = "xxlimited_3_13.Str", + .basicsize = 0, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = Str_Type_slots, +}; + + +/* Function of two integers returning integer (with C "long int" arithmetic) */ + +PyDoc_STRVAR(xx_foo_doc, +"foo(i,j)\n\ +\n\ +Return the sum of i and j."); + +static PyObject * +xx_foo(PyObject *module, PyObject *args) +{ + long i, j; + long res; + if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) + return NULL; + res = i+j; /* XXX Do something here */ + return PyLong_FromLong(res); +} + + +/* Function of no arguments returning new Xxo object */ + +static PyObject * +xx_new(PyObject *module, PyObject *Py_UNUSED(unused)) +{ + XxoObject *rv; + + rv = newXxoObject(module); + if (rv == NULL) + return NULL; + return (PyObject *)rv; +} + + + +/* List of functions defined in the module */ + +static PyMethodDef xx_methods[] = { + {"foo", xx_foo, METH_VARARGS, + xx_foo_doc}, + {"new", xx_new, METH_NOARGS, + PyDoc_STR("new() -> new Xx object")}, + {NULL, NULL} /* sentinel */ +}; + + +/* The module itself */ + +PyDoc_STRVAR(module_doc, +"This is a template module just for instruction."); + +static int +xx_modexec(PyObject *m) +{ + xx_state *state = PyModule_GetState(m); + + state->Error_Type = PyErr_NewException("xxlimited_3_13.Error", NULL, NULL); + if (state->Error_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)state->Error_Type) < 0) { + return -1; + } + + state->Xxo_Type = PyType_FromModuleAndSpec(m, &Xxo_Type_spec, NULL); + if (state->Xxo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)state->Xxo_Type) < 0) { + return -1; + } + + // Add the Str type. It is not needed from C code, so it is only + // added to the module dict. + // It does not inherit from "object" (PyObject_Type), but from "str" + // (PyUnincode_Type). + PyObject *Str_Type = PyType_FromModuleAndSpec( + m, &Str_Type_spec, (PyObject *)&PyUnicode_Type); + if (Str_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, (PyTypeObject*)Str_Type) < 0) { + return -1; + } + Py_DECREF(Str_Type); + + return 0; +} + +static PyModuleDef_Slot xx_slots[] = { + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ + {Py_mod_exec, xx_modexec}, + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + + {0, NULL} +}; + +// Module finalization: modules that hold references in their module state +// need to implement the fullowing GC hooks. They're similar to the ones for +// types (see "Xxo finalization"). + +static int +xx_traverse(PyObject *module, visitproc visit, void *arg) +{ + xx_state *state = PyModule_GetState(module); + Py_VISIT(state->Xxo_Type); + Py_VISIT(state->Error_Type); + return 0; +} + +static int +xx_clear(PyObject *module) +{ + xx_state *state = PyModule_GetState(module); + Py_CLEAR(state->Xxo_Type); + Py_CLEAR(state->Error_Type); + return 0; +} + +static void +xx_free(void *module) +{ + // allow xx_modexec to omit calling xx_clear on error + (void)xx_clear((PyObject *)module); +} + +static struct PyModuleDef xxmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "xxlimited_3_13", + .m_doc = module_doc, + .m_size = sizeof(xx_state), + .m_methods = xx_methods, + .m_slots = xx_slots, + .m_traverse = xx_traverse, + .m_clear = xx_clear, + .m_free = xx_free, +}; + + +/* Export function for the module. *Must* be called PyInit_xx; usually it is + * the only non-`static` object in a module definition. + */ + +PyMODINIT_FUNC +PyInit_xxlimited_3_13(void) +{ + return PyModuleDef_Init(&xxmodule); +} diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index 9c5820fbe97a6b0..0a6732835eb51f5 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -858,7 +858,7 @@ save_unconsumed_input(compobject *self, Py_buffer *data, int err) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary zlib.Decompress.decompress cls: defining_class @@ -872,15 +872,15 @@ zlib.Decompress.decompress Return a bytes object containing the decompressed version of the data. -After calling this function, some of the input data may still be stored in -internal buffers for later processing. +After calling this function, some of the input data may still be +stored in internal buffers for later processing. Call the flush() method to clear these buffers. [clinic start generated code]*/ static PyObject * zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=b024a93c2c922d57 input=77de124bd2a2ecc0]*/ +/*[clinic end generated code: output=b024a93c2c922d57 input=9035027c9e4be7fd]*/ { int err = Z_OK; Py_ssize_t ibuflen; @@ -1675,7 +1675,6 @@ decompress(ZlibDecompressor *self, uint8_t *data, } /*[clinic input] -@permit_long_docstring_body zlib._ZlibDecompressor.decompress data: Py_buffer @@ -1683,25 +1682,26 @@ zlib._ZlibDecompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * zlib__ZlibDecompressor_decompress_impl(ZlibDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=ac00dcf73e843e99 input=c9278e791be1152b]*/ +/*[clinic end generated code: output=ac00dcf73e843e99 input=d7862eade3f29d56]*/ { PyObject *result = NULL; diff --git a/Objects/abstract.c b/Objects/abstract.c index 0bbf60840a3346b..28f751965f36b94 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -208,7 +208,7 @@ PyObject_GetItem(PyObject *o, PyObject *key) int PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) { - if (PyDict_CheckExact(obj)) { + if (PyAnyDict_CheckExact(obj)) { return PyDict_GetItemRef(obj, key, result); } @@ -2462,7 +2462,7 @@ PyMapping_Keys(PyObject *o) if (o == NULL) { return null_error(); } - if (PyDict_CheckExact(o)) { + if (PyAnyDict_CheckExact(o)) { return PyDict_Keys(o); } return method_output_as_list(o, &_Py_ID(keys)); @@ -2474,7 +2474,7 @@ PyMapping_Items(PyObject *o) if (o == NULL) { return null_error(); } - if (PyDict_CheckExact(o)) { + if (PyAnyDict_CheckExact(o)) { return PyDict_Items(o); } return method_output_as_list(o, &_Py_ID(items)); @@ -2486,7 +2486,7 @@ PyMapping_Values(PyObject *o) if (o == NULL) { return null_error(); } - if (PyDict_CheckExact(o)) { + if (PyAnyDict_CheckExact(o)) { return PyDict_Values(o); } return method_output_as_list(o, &_Py_ID(values)); @@ -2951,3 +2951,11 @@ PyIter_Send(PyObject *iter, PyObject *arg, PyObject **result) } return PYGEN_ERROR; } + +PySendResultPair +_PyIter_Send(PyObject *iter, PyObject *arg) +{ + PySendResultPair pair; + pair.kind = PyIter_Send(iter, arg, &pair.object); + return pair; +} diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index c583193b5a252ca..696ddad8efaae55 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -402,7 +402,7 @@ bytearray_repeat_lock_held(PyObject *op, Py_ssize_t count) PyByteArrayObject* result = (PyByteArrayObject *)PyByteArray_FromStringAndSize(NULL, size); const char* buf = PyByteArray_AS_STRING(self); if (result != NULL && size != 0) { - _PyBytes_Repeat(result->ob_bytes, size, buf, mysize); + _PyBytes_RepeatBuffer(result->ob_bytes, size, buf, mysize); } return (PyObject *)result; } @@ -439,7 +439,7 @@ bytearray_irepeat_lock_held(PyObject *op, Py_ssize_t count) } char* buf = PyByteArray_AS_STRING(self); - _PyBytes_Repeat(buf, size, buf, mysize); + _PyBytes_RepeatBuffer(buf, size, buf, mysize); return Py_NewRef(self); } @@ -1620,6 +1620,7 @@ bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n) /*[clinic input] +@permit_long_summary @critical_section bytearray.translate @@ -1630,14 +1631,15 @@ bytearray.translate Return a copy with each character mapped by the given translation table. -All characters occurring in the optional argument delete are removed. -The remaining characters are mapped through the given translation table. +All characters occurring in the optional argument delete are +removed. The remaining characters are mapped through the given +translation table. [clinic start generated code]*/ static PyObject * bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, PyObject *deletechars) -/*[clinic end generated code: output=b6a8f01c2a74e446 input=cd6fa93ca04e05bc]*/ +/*[clinic end generated code: output=b6a8f01c2a74e446 input=e30d2ae004365ed9]*/ { char *input, *output; const char *table_chars; @@ -1727,7 +1729,6 @@ bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, /*[clinic input] @permit_long_summary -@permit_long_docstring_body @staticmethod bytearray.maketrans @@ -1737,15 +1738,15 @@ bytearray.maketrans Return a translation table usable for the bytes or bytearray translate method. -The returned table will be one where each byte in frm is mapped to the byte at -the same position in to. +The returned table will be one where each byte in frm is mapped to +the byte at the same position in to. The bytes objects frm and to must be of the same length. [clinic start generated code]*/ static PyObject * bytearray_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=1df267d99f56b15e input=1146b43a592eca13]*/ +/*[clinic end generated code: output=1df267d99f56b15e input=c2f5f6e7e6b0221d]*/ { return _Py_bytes_maketrans(frm, to); } @@ -1785,8 +1786,8 @@ bytearray.split sep: object = None The delimiter according which to split the bytearray. - None (the default value) means split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). + None (the default value) means split on ASCII whitespace + characters (space, tab, return, newline, formfeed, vertical tab). maxsplit: Py_ssize_t = -1 Maximum number of splits to do. -1 (the default value) means no limit. @@ -1797,7 +1798,7 @@ Return a list of the sections in the bytearray, using sep as the delimiter. static PyObject * bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=833e2cf385d9a04d input=dd9f6e2910cc3a34]*/ +/*[clinic end generated code: output=833e2cf385d9a04d input=45605178023b52ac]*/ { PyObject *list = NULL; @@ -1829,7 +1830,6 @@ bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, } /*[clinic input] -@permit_long_docstring_body @critical_section bytearray.partition @@ -1838,17 +1838,18 @@ bytearray.partition Partition the bytearray into three parts using the given separator. -This will search for the separator sep in the bytearray. If the separator is -found, returns a 3-tuple containing the part before the separator, the -separator itself, and the part after it as new bytearray objects. +This will search for the separator sep in the bytearray. If the +separator is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it as new +bytearray objects. -If the separator is not found, returns a 3-tuple containing the copy of the -original bytearray object and two empty bytearray objects. +If the separator is not found, returns a 3-tuple containing the copy +of the original bytearray object and two empty bytearray objects. [clinic start generated code]*/ static PyObject * bytearray_partition_impl(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=b5fa1e03f10cfccb input=b87276af883f39d9]*/ +/*[clinic end generated code: output=b5fa1e03f10cfccb input=d76673ed03acf5dd]*/ { PyObject *bytesep, *result; @@ -1868,7 +1869,6 @@ bytearray_partition_impl(PyByteArrayObject *self, PyObject *sep) } /*[clinic input] -@permit_long_docstring_body @critical_section bytearray.rpartition @@ -1877,18 +1877,19 @@ bytearray.rpartition Partition the bytearray into three parts using the given separator. -This will search for the separator sep in the bytearray, starting at the end. -If the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it as new bytearray -objects. +This will search for the separator sep in the bytearray, starting at +the end. If the separator is found, returns a 3-tuple containing +the part before the separator, the separator itself, and the part +after it as new bytearray objects. -If the separator is not found, returns a 3-tuple containing two empty bytearray -objects and the copy of the original bytearray object. +If the separator is not found, returns a 3-tuple containing two +empty bytearray objects and the copy of the original bytearray +object. [clinic start generated code]*/ static PyObject * bytearray_rpartition_impl(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=0186ce7b1ef61289 input=5bdcfc4c333bcfab]*/ +/*[clinic end generated code: output=0186ce7b1ef61289 input=b9216a2074174a36]*/ { PyObject *bytesep, *result; @@ -1909,19 +1910,19 @@ bytearray_rpartition_impl(PyByteArrayObject *self, PyObject *sep) /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section bytearray.rsplit = bytearray.split Return a list of the sections in the bytearray, using sep as the delimiter. -Splitting is done starting at the end of the bytearray and working to the front. +Splitting is done starting at the end of the bytearray and working +to the front. [clinic start generated code]*/ static PyObject * bytearray_rsplit_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=a55e0b5a03cb6190 input=60e9abf305128ff4]*/ +/*[clinic end generated code: output=a55e0b5a03cb6190 input=e201671c9a0c19ee]*/ { PyObject *list = NULL; @@ -2392,7 +2393,6 @@ bytearray_strip_impl_helper(PyByteArrayObject* self, PyObject* bytes, int stript } /*[clinic input] -@permit_long_docstring_body @critical_section bytearray.strip @@ -2401,12 +2401,13 @@ bytearray.strip Strip leading and trailing bytes contained in the argument. -If the argument is omitted or None, strip leading and trailing ASCII whitespace. +If the argument is omitted or None, strip leading and trailing ASCII +whitespace. [clinic start generated code]*/ static PyObject * bytearray_strip_impl(PyByteArrayObject *self, PyObject *bytes) -/*[clinic end generated code: output=760412661a34ad5a input=6acaf88b2ec9daa7]*/ +/*[clinic end generated code: output=760412661a34ad5a input=f4ec5fa609df7d14]*/ { return bytearray_strip_impl_helper(self, bytes, BOTHSTRIP); } @@ -2506,11 +2507,11 @@ bytearray.decode encoding: str(c_default="NULL") = 'utf-8' The encoding with which to decode the bytearray. errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for the handling of decoding errors. - The default is 'strict' meaning that decoding errors raise a - UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that - can handle UnicodeDecodeErrors. + The error handling scheme to use for the handling of decoding + errors. The default is 'strict' meaning that decoding errors + raise a UnicodeDecodeError. Other possible values are 'ignore' + and 'replace' as well as any other name registered with + codecs.register_error that can handle UnicodeDecodeErrors. Decode the bytearray using the codec registered for encoding. [clinic start generated code]*/ @@ -2518,7 +2519,7 @@ Decode the bytearray using the codec registered for encoding. static PyObject * bytearray_decode_impl(PyByteArrayObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=f57d43f4a00b42c5 input=86c303ee376b8453]*/ +/*[clinic end generated code: output=f57d43f4a00b42c5 input=e51ce9b82b51e2ca]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -2550,14 +2551,15 @@ bytearray.join Concatenate any number of bytes/bytearray objects. -The bytearray whose method is called is inserted in between each pair. +The bytearray whose method is called is inserted in between each +pair. The result is returned as a new bytearray object. [clinic start generated code]*/ static PyObject * bytearray_join_impl(PyByteArrayObject *self, PyObject *iterable_of_bytes) -/*[clinic end generated code: output=0ced382b5846a7ee input=49627e07ca31ca26]*/ +/*[clinic end generated code: output=0ced382b5846a7ee input=0a31db349efcd7fa]*/ { PyObject *ret; self->ob_exports++; // this protects `self` from being cleared/resized if `iterable_of_bytes` is a custom iterator @@ -2588,7 +2590,6 @@ bytearray_rjust(PyObject *self, PyObject *const *args, Py_ssize_t nargs) /*[clinic input] @permit_long_summary -@permit_long_docstring_body @critical_section bytearray.splitlines @@ -2596,13 +2597,13 @@ bytearray.splitlines Return a list of the lines in the bytearray, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) -/*[clinic end generated code: output=4223c94b895f6ad9 input=21bc3f02bf1be832]*/ +/*[clinic end generated code: output=4223c94b895f6ad9 input=cc2bb740eed19f27]*/ { return stringlib_splitlines( (PyObject*) self, PyByteArray_AS_STRING(self), @@ -2620,12 +2621,13 @@ bytearray.fromhex Create a bytearray object from a string of hexadecimal numbers. Spaces between two numbers are accepted. -Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') +Example: + bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') [clinic start generated code]*/ static PyObject * bytearray_fromhex_impl(PyTypeObject *type, PyObject *string) -/*[clinic end generated code: output=8f0f0b6d30fb3ba0 input=7e314e5b2d7ab484]*/ +/*[clinic end generated code: output=8f0f0b6d30fb3ba0 input=2243a8b0b9e66cd5]*/ { PyObject *result = _PyBytes_FromHex(string, type == &PyByteArray_Type); if (type != &PyByteArray_Type && result != NULL) { @@ -2641,8 +2643,8 @@ bytearray.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: Py_ssize_t = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Create a string of hexadecimal numbers from a bytearray object. @@ -2661,7 +2663,7 @@ Create a string of hexadecimal numbers from a bytearray object. static PyObject * bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t bytes_per_sep) -/*[clinic end generated code: output=c9563921aff1262b input=d2b23ef057cfcad5]*/ +/*[clinic end generated code: output=c9563921aff1262b input=9ed746203691e894]*/ { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index 56a461d0dd08a78..414afeb7bb003c7 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -277,8 +277,8 @@ _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len) PyDoc_STRVAR_shared(_Py_title__doc__, "B.title() -> copy of B\n\ \n\ -Return a titlecased version of B, i.e. ASCII words start with uppercase\n\ -characters, all remaining cased characters have lowercase."); +Return a titlecased version of B, i.e. ASCII words start with\n\ +uppercase characters, all remaining cased characters have lowercase."); void _Py_bytes_title(char *result, const char *s, Py_ssize_t len) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8a9d1b133affb3e..f63185e14284b1a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3,7 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" // _Py_bytes_startswith() -#include "pycore_bytesobject.h" // _PyBytes_Find(), _PyBytes_Repeat() +#include "pycore_bytesobject.h" // _PyBytes_Find(), _PyBytes_RepeatBuffer() #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_format.h" // F_LJUST @@ -11,6 +11,7 @@ #include "pycore_global_objects.h"// _Py_GET_GLOBAL_OBJECT() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _PyLong_DigitValue +#include "pycore_list.h" // _PyList_GetItemRef #include "pycore_object.h" // _PyObject_GC_TRACK #include "pycore_pymem.h" // PYMEM_CLEANBYTE #include "pycore_strhex.h" // _Py_strhex_with_sep() @@ -1581,8 +1582,8 @@ _PyBytes_Concat(PyObject *a, PyObject *b) return result; } -static PyObject * -bytes_repeat(PyObject *self, Py_ssize_t n) +PyObject * +_PyBytes_Repeat(PyObject *self, Py_ssize_t n) { PyBytesObject *a = _PyBytes_CAST(self); if (n < 0) @@ -1613,7 +1614,7 @@ bytes_repeat(PyObject *self, Py_ssize_t n) set_ob_shash(op, -1); op->ob_sval[size] = '\0'; - _PyBytes_Repeat(op->ob_sval, size, a->ob_sval, Py_SIZE(a)); + _PyBytes_RepeatBuffer(op->ob_sval, size, a->ob_sval, Py_SIZE(a)); return (PyObject *) op; } @@ -1805,7 +1806,7 @@ bytes_buffer_getbuffer(PyObject *op, Py_buffer *view, int flags) static PySequenceMethods bytes_as_sequence = { bytes_length, /*sq_length*/ _PyBytes_Concat, /*sq_concat*/ - bytes_repeat, /*sq_repeat*/ + _PyBytes_Repeat, /*sq_repeat*/ bytes_item, /*sq_item*/ 0, /*sq_slice*/ 0, /*sq_ass_item*/ @@ -1848,12 +1849,13 @@ bytes___bytes___impl(PyBytesObject *self) #define BOTHSTRIP 2 /*[clinic input] +@permit_long_summary bytes.split sep: object = None The delimiter according which to split the bytes. - None (the default value) means split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). + None (the default value) means split on ASCII whitespace + characters (space, tab, return, newline, formfeed, vertical tab). maxsplit: Py_ssize_t = -1 Maximum number of splits to do. -1 (the default value) means no limit. @@ -1863,7 +1865,7 @@ Return a list of the sections in the bytes, using sep as the delimiter. static PyObject * bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=52126b5844c1d8ef input=8b809b39074abbfa]*/ +/*[clinic end generated code: output=52126b5844c1d8ef input=330ff95d92544b05]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -1885,7 +1887,6 @@ bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) } /*[clinic input] -@permit_long_docstring_body bytes.partition sep: Py_buffer @@ -1893,17 +1894,17 @@ bytes.partition Partition the bytes into three parts using the given separator. -This will search for the separator sep in the bytes. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. +This will search for the separator sep in the bytes. If the +separator is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it. -If the separator is not found, returns a 3-tuple containing the original bytes -object and two empty bytes objects. +If the separator is not found, returns a 3-tuple containing the +original bytes object and two empty bytes objects. [clinic start generated code]*/ static PyObject * bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=f532b392a17ff695 input=31c55a0cebaf7722]*/ +/*[clinic end generated code: output=f532b392a17ff695 input=2e6e551ea4f8b95a]*/ { return stringlib_partition( (PyObject*) self, @@ -1913,7 +1914,6 @@ bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) } /*[clinic input] -@permit_long_docstring_body bytes.rpartition sep: Py_buffer @@ -1921,17 +1921,18 @@ bytes.rpartition Partition the bytes into three parts using the given separator. -This will search for the separator sep in the bytes, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. +This will search for the separator sep in the bytes, starting at the +end. If the separator is found, returns a 3-tuple containing the +part before the separator, the separator itself, and the part after +it. -If the separator is not found, returns a 3-tuple containing two empty bytes -objects and the original bytes object. +If the separator is not found, returns a 3-tuple containing two +empty bytes objects and the original bytes object. [clinic start generated code]*/ static PyObject * bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=191b114cbb028e50 input=9ea5a3ab0b02bf52]*/ +/*[clinic end generated code: output=191b114cbb028e50 input=f7d24f722a5470a4]*/ { return stringlib_rpartition( (PyObject*) self, @@ -1941,17 +1942,18 @@ bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary bytes.rsplit = bytes.split Return a list of the sections in the bytes, using sep as the delimiter. -Splitting is done starting at the end of the bytes and working to the front. +Splitting is done starting at the end of the bytes and working to +the front. [clinic start generated code]*/ static PyObject * bytes_rsplit_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=ba698d9ea01e1c8f input=55b6eaea1f3d7046]*/ +/*[clinic end generated code: output=ba698d9ea01e1c8f input=ba9bee56285f43e4]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -2171,7 +2173,6 @@ do_argstrip(PyBytesObject *self, int striptype, PyObject *bytes) } /*[clinic input] -@permit_long_docstring_body bytes.strip bytes: object = None @@ -2179,12 +2180,13 @@ bytes.strip Strip leading and trailing bytes contained in the argument. -If the argument is omitted or None, strip leading and trailing ASCII whitespace. +If the argument is omitted or None, strip leading and trailing ASCII +whitespace. [clinic start generated code]*/ static PyObject * bytes_strip_impl(PyBytesObject *self, PyObject *bytes) -/*[clinic end generated code: output=c7c228d3bd104a1b input=71904cd278c0ee03]*/ +/*[clinic end generated code: output=c7c228d3bd104a1b input=9ffea5f752032bd0]*/ { return do_argstrip(self, BOTHSTRIP, bytes); } @@ -2244,6 +2246,7 @@ bytes_count_impl(PyBytesObject *self, PyObject *sub, Py_ssize_t start, /*[clinic input] +@permit_long_summary bytes.translate table: object @@ -2253,14 +2256,15 @@ bytes.translate Return a copy with each character mapped by the given translation table. -All characters occurring in the optional argument delete are removed. -The remaining characters are mapped through the given translation table. +All characters occurring in the optional argument delete are +removed. The remaining characters are mapped through the given +translation table. [clinic start generated code]*/ static PyObject * bytes_translate_impl(PyBytesObject *self, PyObject *table, PyObject *deletechars) -/*[clinic end generated code: output=43be3437f1956211 input=0ecdf159f654233c]*/ +/*[clinic end generated code: output=43be3437f1956211 input=bddcdef0a87895d2]*/ { const char *input; char *output; @@ -2378,7 +2382,6 @@ bytes_translate_impl(PyBytesObject *self, PyObject *table, /*[clinic input] @permit_long_summary -@permit_long_docstring_body @staticmethod bytes.maketrans @@ -2388,15 +2391,15 @@ bytes.maketrans Return a translation table usable for the bytes or bytearray translate method. -The returned table will be one where each byte in frm is mapped to the byte at -the same position in to. +The returned table will be one where each byte in frm is mapped to +the byte at the same position in to. The bytes objects frm and to must be of the same length. [clinic start generated code]*/ static PyObject * bytes_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=a36f6399d4b77f6f input=a06b75f44d933fb3]*/ +/*[clinic end generated code: output=a36f6399d4b77f6f input=3a577e5badfea8f7]*/ { return _Py_bytes_maketrans(frm, to); } @@ -2431,6 +2434,7 @@ bytes_replace_impl(PyBytesObject *self, Py_buffer *old, Py_buffer *new, /** End DALKE **/ /*[clinic input] +@permit_long_summary bytes.removeprefix as bytes_removeprefix prefix: Py_buffer @@ -2438,13 +2442,14 @@ bytes.removeprefix as bytes_removeprefix Return a bytes object with the given prefix string removed if present. -If the bytes starts with the prefix string, return bytes[len(prefix):]. -Otherwise, return a copy of the original bytes. +If the bytes starts with the prefix string, return +bytes[len(prefix):]. Otherwise, return a copy of the original +bytes. [clinic start generated code]*/ static PyObject * bytes_removeprefix_impl(PyBytesObject *self, Py_buffer *prefix) -/*[clinic end generated code: output=f006865331a06ab6 input=0c93bac817a8502c]*/ +/*[clinic end generated code: output=f006865331a06ab6 input=3a2672bcee61d7a7]*/ { const char *self_start = PyBytes_AS_STRING(self); Py_ssize_t self_len = PyBytes_GET_SIZE(self); @@ -2467,6 +2472,7 @@ bytes_removeprefix_impl(PyBytesObject *self, Py_buffer *prefix) } /*[clinic input] +@permit_long_summary bytes.removesuffix as bytes_removesuffix suffix: Py_buffer @@ -2474,14 +2480,14 @@ bytes.removesuffix as bytes_removesuffix Return a bytes object with the given suffix string removed if present. -If the bytes ends with the suffix string and that suffix is not empty, -return bytes[:-len(prefix)]. Otherwise, return a copy of the original -bytes. +If the bytes ends with the suffix string and that suffix is not +empty, return bytes[:-len(prefix)]. Otherwise, return a copy of the +original bytes. [clinic start generated code]*/ static PyObject * bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) -/*[clinic end generated code: output=d887d308e3242eeb input=9f4e1da8c637bbf1]*/ +/*[clinic end generated code: output=d887d308e3242eeb input=04df5f18a36f69d7]*/ { const char *self_start = PyBytes_AS_STRING(self); Py_ssize_t self_len = PyBytes_GET_SIZE(self); @@ -2561,11 +2567,11 @@ bytes.decode encoding: str(c_default="NULL") = 'utf-8' The encoding with which to decode the bytes. errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for the handling of decoding errors. - The default is 'strict' meaning that decoding errors raise a - UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that - can handle UnicodeDecodeErrors. + The error handling scheme to use for the handling of decoding + errors. The default is 'strict' meaning that decoding errors + raise a UnicodeDecodeError. Other possible values are 'ignore' + and 'replace' as well as any other name registered with + codecs.register_error that can handle UnicodeDecodeErrors. Decode the bytes using the codec registered for encoding. [clinic start generated code]*/ @@ -2573,27 +2579,27 @@ Decode the bytes using the codec registered for encoding. static PyObject * bytes_decode_impl(PyBytesObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=5649a53dde27b314 input=958174769d2a40ca]*/ +/*[clinic end generated code: output=5649a53dde27b314 input=94e9b8524f1d7f37]*/ { return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary bytes.splitlines keepends: bool = False Return a list of the lines in the bytes, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * bytes_splitlines_impl(PyBytesObject *self, int keepends) -/*[clinic end generated code: output=3484149a5d880ffb input=d17968d2a355fe55]*/ +/*[clinic end generated code: output=3484149a5d880ffb input=8734672f34430514]*/ { return stringlib_splitlines( (PyObject*) self, PyBytes_AS_STRING(self), @@ -2744,8 +2750,8 @@ bytes.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: Py_ssize_t = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Create a string of hexadecimal numbers from a bytes object. @@ -2763,7 +2769,7 @@ Create a string of hexadecimal numbers from a bytes object. static PyObject * bytes_hex_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t bytes_per_sep) -/*[clinic end generated code: output=588821f02cb9d8f5 input=bd8eceb755d8230f]*/ +/*[clinic end generated code: output=588821f02cb9d8f5 input=b8d40cf203d172dc]*/ { const char *argbuf = PyBytes_AS_STRING(self); Py_ssize_t arglen = PyBytes_GET_SIZE(self); @@ -2991,8 +2997,10 @@ _PyBytes_FromList(PyObject *x) size = _PyBytesWriter_GetAllocated(writer); for (Py_ssize_t i = 0; i < PyList_GET_SIZE(x); i++) { - PyObject *item = PyList_GET_ITEM(x, i); - Py_INCREF(item); + PyObject *item = _PyList_GetItemRef((PyListObject *)x, i); + if (item == NULL) { + goto error; + } Py_ssize_t value = PyNumber_AsSsize_t(item, NULL); Py_DECREF(item); if (value == -1 && PyErr_Occurred()) @@ -3555,7 +3563,7 @@ bytes_iter(PyObject *seq) void -_PyBytes_Repeat(char* dest, Py_ssize_t len_dest, +_PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src) { if (len_dest == 0) { diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 64603adcc1124b1..41ce82c05c57d97 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -679,8 +679,9 @@ PyDoc_STRVAR(bytearray_translate__doc__, " table\n" " Translation table, which must be a bytes object of length 256.\n" "\n" -"All characters occurring in the optional argument delete are removed.\n" -"The remaining characters are mapped through the given translation table."); +"All characters occurring in the optional argument delete are\n" +"removed. The remaining characters are mapped through the given\n" +"translation table."); #define BYTEARRAY_TRANSLATE_METHODDEF \ {"translate", _PyCFunction_CAST(bytearray_translate), METH_FASTCALL|METH_KEYWORDS, bytearray_translate__doc__}, @@ -750,8 +751,8 @@ PyDoc_STRVAR(bytearray_maketrans__doc__, "\n" "Return a translation table usable for the bytes or bytearray translate method.\n" "\n" -"The returned table will be one where each byte in frm is mapped to the byte at\n" -"the same position in to.\n" +"The returned table will be one where each byte in frm is mapped to\n" +"the byte at the same position in to.\n" "\n" "The bytes objects frm and to must be of the same length."); @@ -901,8 +902,8 @@ PyDoc_STRVAR(bytearray_split__doc__, "\n" " sep\n" " The delimiter according which to split the bytearray.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit."); @@ -991,12 +992,13 @@ PyDoc_STRVAR(bytearray_partition__doc__, "\n" "Partition the bytearray into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytearray. If the separator is\n" -"found, returns a 3-tuple containing the part before the separator, the\n" -"separator itself, and the part after it as new bytearray objects.\n" +"This will search for the separator sep in the bytearray. If the\n" +"separator is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it as new\n" +"bytearray objects.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the copy of the\n" -"original bytearray object and two empty bytearray objects."); +"If the separator is not found, returns a 3-tuple containing the copy\n" +"of the original bytearray object and two empty bytearray objects."); #define BYTEARRAY_PARTITION_METHODDEF \ {"partition", (PyCFunction)bytearray_partition, METH_O, bytearray_partition__doc__}, @@ -1022,13 +1024,14 @@ PyDoc_STRVAR(bytearray_rpartition__doc__, "\n" "Partition the bytearray into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytearray, starting at the end.\n" -"If the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it as new bytearray\n" -"objects.\n" +"This will search for the separator sep in the bytearray, starting at\n" +"the end. If the separator is found, returns a 3-tuple containing\n" +"the part before the separator, the separator itself, and the part\n" +"after it as new bytearray objects.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty bytearray\n" -"objects and the copy of the original bytearray object."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty bytearray objects and the copy of the original bytearray\n" +"object."); #define BYTEARRAY_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)bytearray_rpartition, METH_O, bytearray_rpartition__doc__}, @@ -1056,13 +1059,14 @@ PyDoc_STRVAR(bytearray_rsplit__doc__, "\n" " sep\n" " The delimiter according which to split the bytearray.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit.\n" "\n" -"Splitting is done starting at the end of the bytearray and working to the front."); +"Splitting is done starting at the end of the bytearray and working\n" +"to the front."); #define BYTEARRAY_RSPLIT_METHODDEF \ {"rsplit", _PyCFunction_CAST(bytearray_rsplit), METH_FASTCALL|METH_KEYWORDS, bytearray_rsplit__doc__}, @@ -1364,7 +1368,8 @@ PyDoc_STRVAR(bytearray_strip__doc__, "\n" "Strip leading and trailing bytes contained in the argument.\n" "\n" -"If the argument is omitted or None, strip leading and trailing ASCII whitespace."); +"If the argument is omitted or None, strip leading and trailing ASCII\n" +"whitespace."); #define BYTEARRAY_STRIP_METHODDEF \ {"strip", _PyCFunction_CAST(bytearray_strip), METH_FASTCALL, bytearray_strip__doc__}, @@ -1475,11 +1480,11 @@ PyDoc_STRVAR(bytearray_decode__doc__, " encoding\n" " The encoding with which to decode the bytearray.\n" " errors\n" -" The error handling scheme to use for the handling of decoding errors.\n" -" The default is \'strict\' meaning that decoding errors raise a\n" -" UnicodeDecodeError. Other possible values are \'ignore\' and \'replace\'\n" -" as well as any other name registered with codecs.register_error that\n" -" can handle UnicodeDecodeErrors."); +" The error handling scheme to use for the handling of decoding\n" +" errors. The default is \'strict\' meaning that decoding errors\n" +" raise a UnicodeDecodeError. Other possible values are \'ignore\'\n" +" and \'replace\' as well as any other name registered with\n" +" codecs.register_error that can handle UnicodeDecodeErrors."); #define BYTEARRAY_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(bytearray_decode), METH_FASTCALL|METH_KEYWORDS, bytearray_decode__doc__}, @@ -1578,7 +1583,8 @@ PyDoc_STRVAR(bytearray_join__doc__, "\n" "Concatenate any number of bytes/bytearray objects.\n" "\n" -"The bytearray whose method is called is inserted in between each pair.\n" +"The bytearray whose method is called is inserted in between each\n" +"pair.\n" "\n" "The result is returned as a new bytearray object."); @@ -1606,8 +1612,8 @@ PyDoc_STRVAR(bytearray_splitlines__doc__, "\n" "Return a list of the lines in the bytearray, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define BYTEARRAY_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(bytearray_splitlines), METH_FASTCALL|METH_KEYWORDS, bytearray_splitlines__doc__}, @@ -1678,7 +1684,8 @@ PyDoc_STRVAR(bytearray_fromhex__doc__, "Create a bytearray object from a string of hexadecimal numbers.\n" "\n" "Spaces between two numbers are accepted.\n" -"Example: bytearray.fromhex(\'B9 01EF\') -> bytearray(b\'\\\\xb9\\\\x01\\\\xef\')"); +"Example:\n" +" bytearray.fromhex(\'B9 01EF\') -> bytearray(b\'\\\\xb9\\\\x01\\\\xef\')"); #define BYTEARRAY_FROMHEX_METHODDEF \ {"fromhex", (PyCFunction)bytearray_fromhex, METH_O|METH_CLASS, bytearray_fromhex__doc__}, @@ -1705,8 +1712,8 @@ PyDoc_STRVAR(bytearray_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = bytearray([0xb9, 0x01, 0xef])\n" @@ -1875,4 +1882,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=2cacb323147202b9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6dc315d35de3e670 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 4ff696be91b12de..ee2b737f9e63f97 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -35,8 +35,8 @@ PyDoc_STRVAR(bytes_split__doc__, "\n" " sep\n" " The delimiter according which to split the bytes.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit."); @@ -122,12 +122,12 @@ PyDoc_STRVAR(bytes_partition__doc__, "\n" "Partition the bytes into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytes. If the separator is found,\n" -"returns a 3-tuple containing the part before the separator, the separator\n" -"itself, and the part after it.\n" +"This will search for the separator sep in the bytes. If the\n" +"separator is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the original bytes\n" -"object and two empty bytes objects."); +"If the separator is not found, returns a 3-tuple containing the\n" +"original bytes object and two empty bytes objects."); #define BYTES_PARTITION_METHODDEF \ {"partition", (PyCFunction)bytes_partition, METH_O, bytes_partition__doc__}, @@ -161,12 +161,13 @@ PyDoc_STRVAR(bytes_rpartition__doc__, "\n" "Partition the bytes into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytes, starting at the end. If\n" -"the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it.\n" +"This will search for the separator sep in the bytes, starting at the\n" +"end. If the separator is found, returns a 3-tuple containing the\n" +"part before the separator, the separator itself, and the part after\n" +"it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty bytes\n" -"objects and the original bytes object."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty bytes objects and the original bytes object."); #define BYTES_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)bytes_rpartition, METH_O, bytes_rpartition__doc__}, @@ -202,13 +203,14 @@ PyDoc_STRVAR(bytes_rsplit__doc__, "\n" " sep\n" " The delimiter according which to split the bytes.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit.\n" "\n" -"Splitting is done starting at the end of the bytes and working to the front."); +"Splitting is done starting at the end of the bytes and working to\n" +"the front."); #define BYTES_RSPLIT_METHODDEF \ {"rsplit", _PyCFunction_CAST(bytes_rsplit), METH_FASTCALL|METH_KEYWORDS, bytes_rsplit__doc__}, @@ -523,7 +525,8 @@ PyDoc_STRVAR(bytes_strip__doc__, "\n" "Strip leading and trailing bytes contained in the argument.\n" "\n" -"If the argument is omitted or None, strip leading and trailing ASCII whitespace."); +"If the argument is omitted or None, strip leading and trailing ASCII\n" +"whitespace."); #define BYTES_STRIP_METHODDEF \ {"strip", _PyCFunction_CAST(bytes_strip), METH_FASTCALL, bytes_strip__doc__}, @@ -677,8 +680,9 @@ PyDoc_STRVAR(bytes_translate__doc__, " table\n" " Translation table, which must be a bytes object of length 256.\n" "\n" -"All characters occurring in the optional argument delete are removed.\n" -"The remaining characters are mapped through the given translation table."); +"All characters occurring in the optional argument delete are\n" +"removed. The remaining characters are mapped through the given\n" +"translation table."); #define BYTES_TRANSLATE_METHODDEF \ {"translate", _PyCFunction_CAST(bytes_translate), METH_FASTCALL|METH_KEYWORDS, bytes_translate__doc__}, @@ -746,8 +750,8 @@ PyDoc_STRVAR(bytes_maketrans__doc__, "\n" "Return a translation table usable for the bytes or bytearray translate method.\n" "\n" -"The returned table will be one where each byte in frm is mapped to the byte at\n" -"the same position in to.\n" +"The returned table will be one where each byte in frm is mapped to\n" +"the byte at the same position in to.\n" "\n" "The bytes objects frm and to must be of the same length."); @@ -893,8 +897,9 @@ PyDoc_STRVAR(bytes_removeprefix__doc__, "\n" "Return a bytes object with the given prefix string removed if present.\n" "\n" -"If the bytes starts with the prefix string, return bytes[len(prefix):].\n" -"Otherwise, return a copy of the original bytes."); +"If the bytes starts with the prefix string, return\n" +"bytes[len(prefix):]. Otherwise, return a copy of the original\n" +"bytes."); #define BYTES_REMOVEPREFIX_METHODDEF \ {"removeprefix", (PyCFunction)bytes_removeprefix, METH_O, bytes_removeprefix__doc__}, @@ -928,9 +933,9 @@ PyDoc_STRVAR(bytes_removesuffix__doc__, "\n" "Return a bytes object with the given suffix string removed if present.\n" "\n" -"If the bytes ends with the suffix string and that suffix is not empty,\n" -"return bytes[:-len(prefix)]. Otherwise, return a copy of the original\n" -"bytes."); +"If the bytes ends with the suffix string and that suffix is not\n" +"empty, return bytes[:-len(prefix)]. Otherwise, return a copy of the\n" +"original bytes."); #define BYTES_REMOVESUFFIX_METHODDEF \ {"removesuffix", (PyCFunction)bytes_removesuffix, METH_O, bytes_removesuffix__doc__}, @@ -1069,11 +1074,11 @@ PyDoc_STRVAR(bytes_decode__doc__, " encoding\n" " The encoding with which to decode the bytes.\n" " errors\n" -" The error handling scheme to use for the handling of decoding errors.\n" -" The default is \'strict\' meaning that decoding errors raise a\n" -" UnicodeDecodeError. Other possible values are \'ignore\' and \'replace\'\n" -" as well as any other name registered with codecs.register_error that\n" -" can handle UnicodeDecodeErrors."); +" The error handling scheme to use for the handling of decoding\n" +" errors. The default is \'strict\' meaning that decoding errors\n" +" raise a UnicodeDecodeError. Other possible values are \'ignore\'\n" +" and \'replace\' as well as any other name registered with\n" +" codecs.register_error that can handle UnicodeDecodeErrors."); #define BYTES_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(bytes_decode), METH_FASTCALL|METH_KEYWORDS, bytes_decode__doc__}, @@ -1170,8 +1175,8 @@ PyDoc_STRVAR(bytes_splitlines__doc__, "\n" "Return a list of the lines in the bytes, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define BYTES_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(bytes_splitlines), METH_FASTCALL|METH_KEYWORDS, bytes_splitlines__doc__}, @@ -1267,8 +1272,8 @@ PyDoc_STRVAR(bytes_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = b\'\\xb9\\x01\\xef\'\n" @@ -1450,4 +1455,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b252801ff04a89b3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c20458db7a2123db input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 0cd6e0b56620e85..88333e9d3363eb4 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -414,7 +414,8 @@ PyDoc_STRVAR(code__varname_from_oparg__doc__, "\n" "(internal-only) Return the local variable name for the given oparg.\n" "\n" -"WARNING: this method is for internal use only and may change or go away."); +"WARNING: this method is for internal use only and may change or go\n" +"away."); #define CODE__VARNAME_FROM_OPARG_METHODDEF \ {"_varname_from_oparg", _PyCFunction_CAST(code__varname_from_oparg), METH_FASTCALL|METH_KEYWORDS, code__varname_from_oparg__doc__}, @@ -470,4 +471,4 @@ code__varname_from_oparg(PyObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=c5c6e40fc357defe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c22e29e430401b4 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/exceptions.c.h b/Objects/clinic/exceptions.c.h index 9baac8b1cc660b0..5047a673e579c66 100644 --- a/Objects/clinic/exceptions.c.h +++ b/Objects/clinic/exceptions.c.h @@ -44,7 +44,7 @@ BaseException___setstate__(PyObject *self, PyObject *state) { PyObject *return_value = NULL; - Py_BEGIN_CRITICAL_SECTION(self); + Py_BEGIN_CRITICAL_SECTION(state); return_value = BaseException___setstate___impl((PyBaseExceptionObject *)self, state); Py_END_CRITICAL_SECTION(); @@ -380,4 +380,4 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *matcher_value) return return_value; } -/*[clinic end generated code: output=fcf70b3b71f3d14a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e63b88d0443b4f92 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index c0ae9d3ff9b8d3c..8768555c909257a 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -291,8 +291,8 @@ PyDoc_STRVAR(float___getformat____doc__, "It exists mainly to be used in Python\'s test suite.\n" "\n" "This function returns whichever of \'IEEE, big-endian\' or \'IEEE,\n" -"little-endian\' best describes the format of floating-point numbers used by the\n" -"C type named by typestr."); +"little-endian\' best describes the format of floating-point numbers\n" +"used by the C type named by typestr."); #define FLOAT___GETFORMAT___METHODDEF \ {"__getformat__", (PyCFunction)float___getformat__, METH_O|METH_CLASS, float___getformat____doc__}, @@ -353,4 +353,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=f0b2af257213c8b0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5d7b0bf9e47ff997 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 26ba5b954336da5..f3821ef5f70c21c 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -200,11 +200,11 @@ PyDoc_STRVAR(list_sort__doc__, "\n" "Sort the list in ascending order and return None.\n" "\n" -"The sort is in-place (i.e. the list itself is modified) and stable (i.e. the\n" -"order of two equal elements is maintained).\n" +"The sort is in-place (i.e. the list itself is modified) and stable\n" +"(i.e. the order of two equal elements is maintained).\n" "\n" -"If a key function is given, apply it once to each list item and sort them,\n" -"ascending or descending, according to their function values.\n" +"If a key function is given, apply it once to each list item and sort\n" +"them, ascending or descending, according to their function values.\n" "\n" "The reverse flag can be set to sort in descending order."); @@ -468,4 +468,4 @@ list___reversed__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl((PyListObject *)self); } -/*[clinic end generated code: output=ae13fc2b56dc27c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06c21b0bffbe8d84 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index c88772030ec283d..52ecaffa1f4cf35 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -262,19 +262,20 @@ PyDoc_STRVAR(int_to_bytes__doc__, "Return an array of bytes representing an integer.\n" "\n" " length\n" -" Length of bytes object to use. An OverflowError is raised if the\n" -" integer is not representable with the given number of bytes. Default\n" -" is length 1.\n" +" Length of bytes object to use. An OverflowError is raised if\n" +" the integer is not representable with the given number of bytes.\n" +" Default is length 1.\n" " byteorder\n" -" The byte order used to represent the integer. If byteorder is \'big\',\n" -" the most significant byte is at the beginning of the byte array. If\n" -" byteorder is \'little\', the most significant byte is at the end of the\n" -" byte array. To request the native byte order of the host system, use\n" -" sys.byteorder as the byte order value. Default is to use \'big\'.\n" +" The byte order used to represent the integer. If byteorder is\n" +" \'big\', the most significant byte is at the beginning of the byte\n" +" array. If byteorder is \'little\', the most significant byte is at\n" +" the end of the byte array. To request the native byte order of\n" +" the host system, use sys.byteorder as the byte order value.\n" +" Default is to use \'big\'.\n" " signed\n" -" Determines whether two\'s complement is used to represent the integer.\n" -" If signed is False and a negative integer is given, an OverflowError\n" -" is raised."); +" Determines whether two\'s complement is used to represent the\n" +" integer. If signed is False and a negative integer is given,\n" +" an OverflowError is raised."); #define INT_TO_BYTES_METHODDEF \ {"to_bytes", _PyCFunction_CAST(int_to_bytes), METH_FASTCALL|METH_KEYWORDS, int_to_bytes__doc__}, @@ -383,17 +384,19 @@ PyDoc_STRVAR(int_from_bytes__doc__, "\n" " bytes\n" " Holds the array of bytes to convert. The argument must either\n" -" support the buffer protocol or be an iterable object producing bytes.\n" -" Bytes and bytearray are examples of built-in objects that support the\n" -" buffer protocol.\n" +" support the buffer protocol or be an iterable object producing\n" +" bytes. Bytes and bytearray are examples of built-in objects that\n" +" support the buffer protocol.\n" " byteorder\n" -" The byte order used to represent the integer. If byteorder is \'big\',\n" -" the most significant byte is at the beginning of the byte array. If\n" -" byteorder is \'little\', the most significant byte is at the end of the\n" -" byte array. To request the native byte order of the host system, use\n" -" sys.byteorder as the byte order value. Default is to use \'big\'.\n" +" The byte order used to represent the integer. If byteorder is\n" +" \'big\', the most significant byte is at the beginning of the byte\n" +" array. If byteorder is \'little\', the most significant byte is at\n" +" the end of the byte array. To request the native byte order of\n" +" the host system, use sys.byteorder as the byte order value.\n" +" Default is to use \'big\'.\n" " signed\n" -" Indicates whether two\'s complement is used to represent the integer."); +" Indicates whether two\'s complement is used to represent the\n" +" integer."); #define INT_FROM_BYTES_METHODDEF \ {"from_bytes", _PyCFunction_CAST(int_from_bytes), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, int_from_bytes__doc__}, @@ -490,4 +493,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=e68f4e23ead3f649 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d95766fb7ff46963 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index d97c626532c803e..a0cf3243edc08a0 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -259,11 +259,12 @@ PyDoc_STRVAR(memoryview_tobytes__doc__, "\n" "Return the data in the buffer as a byte string.\n" "\n" -"Order can be {\'C\', \'F\', \'A\'}. When order is \'C\' or \'F\', the data of the\n" -"original array is converted to C or Fortran order. For contiguous views,\n" -"\'A\' returns an exact copy of the physical memory. In particular, in-memory\n" -"Fortran order is preserved. For non-contiguous views, the data is converted\n" -"to C first. order=None is the same as order=\'C\'."); +"Order can be {\'C\', \'F\', \'A\'}. When order is \'C\' or \'F\', the data of\n" +"the original array is converted to C or Fortran order. For\n" +"contiguous views, \'A\' returns an exact copy of the physical memory.\n" +"In particular, in-memory Fortran order is preserved. For\n" +"non-contiguous views, the data is converted to C first. order=None\n" +"is the same as order=\'C\'."); #define MEMORYVIEW_TOBYTES_METHODDEF \ {"tobytes", _PyCFunction_CAST(memoryview_tobytes), METH_FASTCALL|METH_KEYWORDS, memoryview_tobytes__doc__}, @@ -348,8 +349,8 @@ PyDoc_STRVAR(memoryview_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = memoryview(b\'\\xb9\\x01\\xef\')\n" @@ -505,4 +506,4 @@ memoryview_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=348b6ddb98a1f412 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3abf9c80cd49229a input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index 894e9be91bbdce9..92129e6444810f4 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -268,7 +268,8 @@ PyDoc_STRVAR(OrderedDict_popitem__doc__, "\n" "Remove and return a (key, value) pair from the dictionary.\n" "\n" -"Pairs are returned in LIFO order if last is true or FIFO order if false."); +"Pairs are returned in LIFO order if last is true or FIFO order if\n" +"false."); #define ORDEREDDICT_POPITEM_METHODDEF \ {"popitem", _PyCFunction_CAST(OrderedDict_popitem), METH_FASTCALL|METH_KEYWORDS, OrderedDict_popitem__doc__}, @@ -451,4 +452,4 @@ OrderedDict_move_to_end(PyObject *self, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=7bc997ca7900f06f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=89f7e92de998f9a4 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/sentinelobject.c.h b/Objects/clinic/sentinelobject.c.h index 51fd35a5979e318..f8503194ae5c740 100644 --- a/Objects/clinic/sentinelobject.c.h +++ b/Objects/clinic/sentinelobject.c.h @@ -2,33 +2,71 @@ preserve [clinic start generated code]*/ -#include "pycore_modsupport.h" // _PyArg_CheckPositional() +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() static PyObject * -sentinel_new_impl(PyTypeObject *type, PyObject *name); +sentinel_new_impl(PyTypeObject *type, PyObject *name, PyObject *repr); static PyObject * sentinel_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - PyTypeObject *base_tp = &PySentinel_Type; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(repr), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "repr", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "sentinel", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *repr = Py_None; - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoKeywords("sentinel", kwargs)) { + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { goto exit; } - if (!_PyArg_CheckPositional("sentinel", PyTuple_GET_SIZE(args), 1, 1)) { + if (!PyUnicode_Check(fastargs[0])) { + _PyArg_BadArgument("sentinel", "argument 1", "str", fastargs[0]); goto exit; } - if (!PyUnicode_Check(PyTuple_GET_ITEM(args, 0))) { - _PyArg_BadArgument("sentinel", "argument 1", "str", PyTuple_GET_ITEM(args, 0)); - goto exit; + name = fastargs[0]; + if (!noptargs) { + goto skip_optional_kwonly; } - name = PyTuple_GET_ITEM(args, 0); - return_value = sentinel_new_impl(type, name); + repr = fastargs[1]; +skip_optional_kwonly: + return_value = sentinel_new_impl(type, name, repr); exit: return return_value; } -/*[clinic end generated code: output=7f28fc0bf0259cba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=958842ece254c82f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 4b53e24fb7d649f..d0753b38843fccf 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -33,8 +33,8 @@ PyDoc_STRVAR(unicode_title__doc__, "\n" "Return a version of the string where each word is titlecased.\n" "\n" -"More specifically, words start with uppercased characters and all remaining\n" -"cased characters have lower case."); +"More specifically, words start with uppercased characters and all\n" +"remaining cased characters have lower case."); #define UNICODE_TITLE_METHODDEF \ {"title", (PyCFunction)unicode_title, METH_NOARGS, unicode_title__doc__}, @@ -54,8 +54,8 @@ PyDoc_STRVAR(unicode_capitalize__doc__, "\n" "Return a capitalized version of the string.\n" "\n" -"More specifically, make the first character have upper case and the rest lower\n" -"case."); +"More specifically, make the first character have upper case and the\n" +"rest lower case."); #define UNICODE_CAPITALIZE_METHODDEF \ {"capitalize", (PyCFunction)unicode_capitalize, METH_NOARGS, unicode_capitalize__doc__}, @@ -93,7 +93,8 @@ PyDoc_STRVAR(unicode_center__doc__, "\n" "Return a centered string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_CENTER_METHODDEF \ {"center", _PyCFunction_CAST(unicode_center), METH_FASTCALL, unicode_center__doc__}, @@ -142,7 +143,8 @@ PyDoc_STRVAR(unicode_count__doc__, "\n" "Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation."); +"Optional arguments start and end are interpreted as in slice\n" +"notation."); #define UNICODE_COUNT_METHODDEF \ {"count", _PyCFunction_CAST(unicode_count), METH_FASTCALL, unicode_count__doc__}, @@ -202,8 +204,8 @@ PyDoc_STRVAR(unicode_encode__doc__, " errors\n" " The error handling scheme to use for encoding errors.\n" " The default is \'strict\' meaning that encoding errors raise a\n" -" UnicodeEncodeError. Other possible values are \'ignore\', \'replace\' and\n" -" \'xmlcharrefreplace\' as well as any other name registered with\n" +" UnicodeEncodeError. Other possible values are \'ignore\', \'replace\'\n" +" and \'xmlcharrefreplace\' as well as any other name registered with\n" " codecs.register_error that can handle UnicodeEncodeErrors."); #define UNICODE_ENCODE_METHODDEF \ @@ -368,8 +370,8 @@ PyDoc_STRVAR(unicode_find__doc__, "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Return -1 on failure."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Return -1 on failure."); #define UNICODE_FIND_METHODDEF \ {"find", _PyCFunction_CAST(unicode_find), METH_FASTCALL, unicode_find__doc__}, @@ -424,8 +426,8 @@ PyDoc_STRVAR(unicode_index__doc__, "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Raises ValueError when the substring is not found."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Raises ValueError when the substring is not found."); #define UNICODE_INDEX_METHODDEF \ {"index", _PyCFunction_CAST(unicode_index), METH_FASTCALL, unicode_index__doc__}, @@ -501,8 +503,8 @@ PyDoc_STRVAR(unicode_islower__doc__, "\n" "Return True if the string is a lowercase string, False otherwise.\n" "\n" -"A string is lowercase if all cased characters in the string are lowercase and\n" -"there is at least one cased character in the string."); +"A string is lowercase if all cased characters in the string are\n" +"lowercase and there is at least one cased character in the string."); #define UNICODE_ISLOWER_METHODDEF \ {"islower", (PyCFunction)unicode_islower, METH_NOARGS, unicode_islower__doc__}, @@ -522,8 +524,8 @@ PyDoc_STRVAR(unicode_isupper__doc__, "\n" "Return True if the string is an uppercase string, False otherwise.\n" "\n" -"A string is uppercase if all cased characters in the string are uppercase and\n" -"there is at least one cased character in the string."); +"A string is uppercase if all cased characters in the string are\n" +"uppercase and there is at least one cased character in the string."); #define UNICODE_ISUPPER_METHODDEF \ {"isupper", (PyCFunction)unicode_isupper, METH_NOARGS, unicode_isupper__doc__}, @@ -564,8 +566,8 @@ PyDoc_STRVAR(unicode_isspace__doc__, "\n" "Return True if the string is a whitespace string, False otherwise.\n" "\n" -"A string is whitespace if all characters in the string are whitespace and there\n" -"is at least one character in the string."); +"A string is whitespace if all characters in the string are\n" +"whitespace and there is at least one character in the string."); #define UNICODE_ISSPACE_METHODDEF \ {"isspace", (PyCFunction)unicode_isspace, METH_NOARGS, unicode_isspace__doc__}, @@ -585,8 +587,8 @@ PyDoc_STRVAR(unicode_isalpha__doc__, "\n" "Return True if the string is an alphabetic string, False otherwise.\n" "\n" -"A string is alphabetic if all characters in the string are alphabetic and there\n" -"is at least one character in the string."); +"A string is alphabetic if all characters in the string are\n" +"alphabetic and there is at least one character in the string."); #define UNICODE_ISALPHA_METHODDEF \ {"isalpha", (PyCFunction)unicode_isalpha, METH_NOARGS, unicode_isalpha__doc__}, @@ -606,8 +608,8 @@ PyDoc_STRVAR(unicode_isalnum__doc__, "\n" "Return True if the string is an alpha-numeric string, False otherwise.\n" "\n" -"A string is alpha-numeric if all characters in the string are alpha-numeric and\n" -"there is at least one character in the string."); +"A string is alpha-numeric if all characters in the string are\n" +"alpha-numeric and there is at least one character in the string."); #define UNICODE_ISALNUM_METHODDEF \ {"isalnum", (PyCFunction)unicode_isalnum, METH_NOARGS, unicode_isalnum__doc__}, @@ -627,8 +629,8 @@ PyDoc_STRVAR(unicode_isdecimal__doc__, "\n" "Return True if the string is a decimal string, False otherwise.\n" "\n" -"A string is a decimal string if all characters in the string are decimal and\n" -"there is at least one character in the string."); +"A string is a decimal string if all characters in the string are\n" +"decimal and there is at least one character in the string."); #define UNICODE_ISDECIMAL_METHODDEF \ {"isdecimal", (PyCFunction)unicode_isdecimal, METH_NOARGS, unicode_isdecimal__doc__}, @@ -648,8 +650,8 @@ PyDoc_STRVAR(unicode_isdigit__doc__, "\n" "Return True if the string is a digit string, False otherwise.\n" "\n" -"A string is a digit string if all characters in the string are digits and there\n" -"is at least one character in the string."); +"A string is a digit string if all characters in the string are\n" +"digits and there is at least one character in the string."); #define UNICODE_ISDIGIT_METHODDEF \ {"isdigit", (PyCFunction)unicode_isdigit, METH_NOARGS, unicode_isdigit__doc__}, @@ -669,8 +671,8 @@ PyDoc_STRVAR(unicode_isnumeric__doc__, "\n" "Return True if the string is a numeric string, False otherwise.\n" "\n" -"A string is numeric if all characters in the string are numeric and there is at\n" -"least one character in the string."); +"A string is numeric if all characters in the string are numeric and\n" +"there is at least one character in the string."); #define UNICODE_ISNUMERIC_METHODDEF \ {"isnumeric", (PyCFunction)unicode_isnumeric, METH_NOARGS, unicode_isnumeric__doc__}, @@ -690,8 +692,8 @@ PyDoc_STRVAR(unicode_isidentifier__doc__, "\n" "Return True if the string is a valid Python identifier, False otherwise.\n" "\n" -"Call keyword.iskeyword(s) to test whether string s is a reserved identifier,\n" -"such as \"def\" or \"class\"."); +"Call keyword.iskeyword(s) to test whether string s is a reserved\n" +"identifier, such as \"def\" or \"class\"."); #define UNICODE_ISIDENTIFIER_METHODDEF \ {"isidentifier", (PyCFunction)unicode_isidentifier, METH_NOARGS, unicode_isidentifier__doc__}, @@ -731,8 +733,8 @@ PyDoc_STRVAR(unicode_join__doc__, "\n" "Concatenate any number of strings.\n" "\n" -"The string whose method is called is inserted in between each given string.\n" -"The result is returned as a new string.\n" +"The string whose method is called is inserted in between each given\n" +"string. The result is returned as a new string.\n" "\n" "Example: \'.\'.join([\'ab\', \'pq\', \'rs\']) -> \'ab.pq.rs\'"); @@ -745,7 +747,8 @@ PyDoc_STRVAR(unicode_ljust__doc__, "\n" "Return a left-justified string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_LJUST_METHODDEF \ {"ljust", _PyCFunction_CAST(unicode_ljust), METH_FASTCALL, unicode_ljust__doc__}, @@ -1008,8 +1011,9 @@ PyDoc_STRVAR(unicode_removeprefix__doc__, "\n" "Return a str with the given prefix string removed if present.\n" "\n" -"If the string starts with the prefix string, return string[len(prefix):].\n" -"Otherwise, return a copy of the original string."); +"If the string starts with the prefix string, return\n" +"string[len(prefix):]. Otherwise, return a copy of the original\n" +"string."); #define UNICODE_REMOVEPREFIX_METHODDEF \ {"removeprefix", (PyCFunction)unicode_removeprefix, METH_O, unicode_removeprefix__doc__}, @@ -1040,9 +1044,9 @@ PyDoc_STRVAR(unicode_removesuffix__doc__, "\n" "Return a str with the given suffix string removed if present.\n" "\n" -"If the string ends with the suffix string and that suffix is not empty,\n" -"return string[:-len(suffix)]. Otherwise, return a copy of the original\n" -"string."); +"If the string ends with the suffix string and that suffix is not\n" +"empty, return string[:-len(suffix)]. Otherwise, return a copy of\n" +"the original string."); #define UNICODE_REMOVESUFFIX_METHODDEF \ {"removesuffix", (PyCFunction)unicode_removesuffix, METH_O, unicode_removesuffix__doc__}, @@ -1073,8 +1077,8 @@ PyDoc_STRVAR(unicode_rfind__doc__, "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Return -1 on failure."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Return -1 on failure."); #define UNICODE_RFIND_METHODDEF \ {"rfind", _PyCFunction_CAST(unicode_rfind), METH_FASTCALL, unicode_rfind__doc__}, @@ -1129,8 +1133,8 @@ PyDoc_STRVAR(unicode_rindex__doc__, "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Raises ValueError when the substring is not found."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Raises ValueError when the substring is not found."); #define UNICODE_RINDEX_METHODDEF \ {"rindex", _PyCFunction_CAST(unicode_rindex), METH_FASTCALL, unicode_rindex__doc__}, @@ -1185,7 +1189,8 @@ PyDoc_STRVAR(unicode_rjust__doc__, "\n" "Return a right-justified string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_RJUST_METHODDEF \ {"rjust", _PyCFunction_CAST(unicode_rjust), METH_FASTCALL, unicode_rjust__doc__}, @@ -1237,18 +1242,18 @@ PyDoc_STRVAR(unicode_split__doc__, " sep\n" " The separator used to split the string.\n" "\n" -" When set to None (the default value), will split on any whitespace\n" -" character (including \\n \\r \\t \\f and spaces) and will discard\n" -" empty strings from the result.\n" +" When set to None (the default value), will split on any\n" +" whitespace character (including \\n \\r \\t \\f and spaces) and\n" +" will discard empty strings from the result.\n" " maxsplit\n" " Maximum number of splits.\n" " -1 (the default value) means no limit.\n" "\n" "Splitting starts at the front of the string and works to the end.\n" "\n" -"Note, str.split() is mainly useful for data that has been intentionally\n" -"delimited. With natural text that includes punctuation, consider using\n" -"the regular expression module."); +"Note, str.split() is mainly useful for data that has been\n" +"intentionally delimited. With natural text that includes\n" +"punctuation, consider using the regular expression module."); #define UNICODE_SPLIT_METHODDEF \ {"split", _PyCFunction_CAST(unicode_split), METH_FASTCALL|METH_KEYWORDS, unicode_split__doc__}, @@ -1331,12 +1336,12 @@ PyDoc_STRVAR(unicode_partition__doc__, "\n" "Partition the string into three parts using the given separator.\n" "\n" -"This will search for the separator in the string. If the separator is found,\n" -"returns a 3-tuple containing the part before the separator, the separator\n" -"itself, and the part after it.\n" +"This will search for the separator in the string. If the separator\n" +"is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the original string\n" -"and two empty strings."); +"If the separator is not found, returns a 3-tuple containing\n" +"the original string and two empty strings."); #define UNICODE_PARTITION_METHODDEF \ {"partition", (PyCFunction)unicode_partition, METH_O, unicode_partition__doc__}, @@ -1347,12 +1352,13 @@ PyDoc_STRVAR(unicode_rpartition__doc__, "\n" "Partition the string into three parts using the given separator.\n" "\n" -"This will search for the separator in the string, starting at the end. If\n" -"the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it.\n" +"This will search for the separator in the string, starting at the\n" +"end. If the separator is found, returns a 3-tuple containing the\n" +"part before the separator, the separator itself, and the part after\n" +"it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty strings\n" -"and the original string."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty strings and the original string."); #define UNICODE_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)unicode_rpartition, METH_O, unicode_rpartition__doc__}, @@ -1366,9 +1372,9 @@ PyDoc_STRVAR(unicode_rsplit__doc__, " sep\n" " The separator used to split the string.\n" "\n" -" When set to None (the default value), will split on any whitespace\n" -" character (including \\n \\r \\t \\f and spaces) and will discard\n" -" empty strings from the result.\n" +" When set to None (the default value), will split on any\n" +" whitespace character (including \\n \\r \\t \\f and spaces) and\n" +" will discard empty strings from the result.\n" " maxsplit\n" " Maximum number of splits.\n" " -1 (the default value) means no limit.\n" @@ -1456,8 +1462,8 @@ PyDoc_STRVAR(unicode_splitlines__doc__, "\n" "Return a list of the lines in the string, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define UNICODE_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(unicode_splitlines), METH_FASTCALL|METH_KEYWORDS, unicode_splitlines__doc__}, @@ -1543,13 +1549,14 @@ PyDoc_STRVAR(unicode_maketrans__doc__, "\n" "Return a translation table usable for str.translate().\n" "\n" -"If there is only one argument, it must be a dictionary mapping Unicode\n" -"ordinals (integers) or characters to Unicode ordinals, strings or None.\n" -"Character keys will be then converted to ordinals.\n" -"If there are two arguments, they must be strings of equal length, and\n" -"in the resulting dictionary, each character in x will be mapped to the\n" -"character at the same position in y. If there is a third argument, it\n" -"must be a string, whose characters will be mapped to None in the result."); +"If there is only one argument, it must be a dictionary mapping\n" +"Unicode ordinals (integers) or characters to Unicode ordinals,\n" +"strings or None. Character keys will be then converted to ordinals.\n" +"If there are two arguments, they must be strings of equal length,\n" +"and in the resulting dictionary, each character in x will be mapped\n" +"to the character at the same position in y. If there is a third\n" +"argument, it must be a string, whose characters will be mapped to\n" +"None in the result."); #define UNICODE_MAKETRANS_METHODDEF \ {"maketrans", _PyCFunction_CAST(unicode_maketrans), METH_FASTCALL|METH_STATIC, unicode_maketrans__doc__}, @@ -1599,12 +1606,13 @@ PyDoc_STRVAR(unicode_translate__doc__, "Replace each character in the string using the given translation table.\n" "\n" " table\n" -" Translation table, which must be a mapping of Unicode ordinals to\n" -" Unicode ordinals, strings, or None.\n" +" Translation table, which must be a mapping of Unicode ordinals\n" +" to Unicode ordinals, strings, or None.\n" "\n" -"The table must implement lookup/indexing via __getitem__, for instance a\n" -"dictionary or list. If this operation raises LookupError, the character is\n" -"left untouched. Characters mapped to None are deleted."); +"The table must implement lookup/indexing via __getitem__, for\n" +"instance a dictionary or list. If this operation raises\n" +"LookupError, the character is left untouched. Characters mapped to\n" +"None are deleted."); #define UNICODE_TRANSLATE_METHODDEF \ {"translate", (PyCFunction)unicode_translate, METH_O, unicode_translate__doc__}, @@ -1908,4 +1916,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=13eaf65699ea9fc9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d243c63e951e31d input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 8be85b1accbdca2..03036020b1cb1ae 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -743,6 +743,10 @@ _PyCode_New(struct _PyCodeConstructor *con) return NULL; } +#ifdef Py_GIL_DISABLED + co->_co_unique_id = _Py_INVALID_UNIQUE_ID; +#endif + if (init_code(co, con) < 0) { Py_DECREF(co); return NULL; @@ -2449,15 +2453,17 @@ code_dealloc(PyObject *self) FT_CLEAR_WEAKREFS(self, co->co_weakreflist); free_monitoring_data(co->_co_monitoring); #ifdef Py_GIL_DISABLED - // The first element always points to the mutable bytecode at the end of - // the code object, which will be freed when the code object is freed. - for (Py_ssize_t i = 1; i < co->co_tlbc->size; i++) { - char *entry = co->co_tlbc->entries[i]; - if (entry != NULL) { - PyMem_Free(entry); + if (co->co_tlbc != NULL) { + // The first element always points to the mutable bytecode at the end of + // the code object, which will be freed when the code object is freed. + for (Py_ssize_t i = 1; i < co->co_tlbc->size; i++) { + char *entry = co->co_tlbc->entries[i]; + if (entry != NULL) { + PyMem_Free(entry); + } } + PyMem_Free(co->co_tlbc); } - PyMem_Free(co->co_tlbc); #endif PyObject_Free(co); } @@ -2840,12 +2846,13 @@ code._varname_from_oparg (internal-only) Return the local variable name for the given oparg. -WARNING: this method is for internal use only and may change or go away. +WARNING: this method is for internal use only and may change or go +away. [clinic start generated code]*/ static PyObject * code__varname_from_oparg_impl(PyCodeObject *self, int oparg) -/*[clinic end generated code: output=1fd1130413184206 input=c5fa3ee9bac7d4ca]*/ +/*[clinic end generated code: output=1fd1130413184206 input=6ba7d6df0d566463]*/ { PyObject *name = PyTuple_GetItem(self->co_localsplusnames, oparg); if (name == NULL) { diff --git a/Objects/descrobject.c b/Objects/descrobject.c index a5926616eeb3cbf..30444b7d6774247 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1178,7 +1178,7 @@ static PyMethodDef mappingproxy_methods[] = { {"copy", mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("mappingproxy objects are generic over two types, signifying (respectively) the types of their keys and values")}, {"__reversed__", mappingproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator")}, {0} diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 42bc63acd9049ce..4b1a7bfe39d3a5f 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -138,7 +138,9 @@ As a consequence of this, split keys have a maximum size of 16. // Forward declarations static PyObject* frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static PyObject* frozendict_new_untracked(PyTypeObject *type); static PyObject* dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static PyObject* dict_new_untracked(PyTypeObject *type); static int dict_merge(PyObject *a, PyObject *b, int override, PyObject **dupkey); static int dict_contains(PyObject *op, PyObject *key); static int dict_merge_from_seq2(PyObject *d, PyObject *seq2, int override); @@ -213,6 +215,15 @@ set_values(PyDictObject *mp, PyDictValues *values) _Py_atomic_store_ptr_release(&mp->ma_values, values); } +// gh-151593: The _Py_LOCK_DONT_DETACH flag ensures that the outer critical +// section is not dropped if there is some contention on the keys lock. +// It also means that it will be important that LOCK_KEYS() is essentially the +// "inner-most" code and that we don't call Py_DECREF() or similar while +// holding the keys lock. +// +// We are not allowed to acquire other locks within LOCK_KEYS(). For example, +// PyType_Modified() must not be called within LOCK_KEYS() since it acquires +// the type lock. #define LOCK_KEYS(keys) PyMutex_LockFlags(&keys->dk_mutex, _Py_LOCK_DONT_DETACH) #define UNLOCK_KEYS(keys) PyMutex_Unlock(&keys->dk_mutex) @@ -784,16 +795,12 @@ _PyDict_CheckConsistency(PyObject *op, int check_content) } -static PyDictKeysObject* -new_keys_object(uint8_t log2_size, bool unicode) +static inline int +get_log2_bytes(uint8_t log2_size) { - Py_ssize_t usable; int log2_bytes; - size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry); - assert(log2_size >= PyDict_LOG_MINSIZE); - usable = USABLE_FRACTION((size_t)1<<log2_size); if (log2_size < 8) { log2_bytes = log2_size; } @@ -809,26 +816,20 @@ new_keys_object(uint8_t log2_size, bool unicode) log2_bytes = log2_size + 2; } - PyDictKeysObject *dk = NULL; - if (log2_size == PyDict_LOG_MINSIZE && unicode) { - dk = _Py_FREELIST_POP_MEM(dictkeys); - } - if (dk == NULL) { - dk = PyMem_Malloc(sizeof(PyDictKeysObject) - + ((size_t)1 << log2_bytes) - + entry_size * usable); - if (dk == NULL) { - PyErr_NoMemory(); - return NULL; - } - } + return log2_bytes; +} + +static inline void +init_keys_object(PyDictKeysObject* dk, uint8_t log2_size, int log2_bytes, int kind, + Py_ssize_t usable, Py_ssize_t entry_size) +{ #ifdef Py_REF_DEBUG _Py_IncRefTotal(_PyThreadState_GET()); #endif dk->dk_refcnt = 1; dk->dk_log2_size = log2_size; dk->dk_log2_index_bytes = log2_bytes; - dk->dk_kind = unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL; + dk->dk_kind = kind; #ifdef Py_GIL_DISABLED dk->dk_mutex = (PyMutex){0}; #endif @@ -837,15 +838,51 @@ new_keys_object(uint8_t log2_size, bool unicode) dk->dk_version = 0; memset(&dk->dk_indices[0], 0xff, ((size_t)1 << log2_bytes)); memset(&dk->dk_indices[(size_t)1 << log2_bytes], 0, entry_size * usable); +} + +static PyDictKeysObject* +new_keys_object(uint8_t log2_size, bool unicode) +{ + Py_ssize_t usable = USABLE_FRACTION((size_t)1<<log2_size); + size_t entry_size = unicode ? sizeof(PyDictUnicodeEntry) : sizeof(PyDictKeyEntry); + + int log2_bytes = get_log2_bytes(log2_size); + + PyDictKeysObject *dk = NULL; + if (log2_size == PyDict_LOG_MINSIZE && unicode) { + dk = _Py_FREELIST_POP_MEM(dictkeys); + } + if (dk == NULL) { + dk = PyMem_Malloc(sizeof(PyDictKeysObject) + + ((size_t)1 << log2_bytes) + + entry_size * usable); + if (dk == NULL) { + PyErr_NoMemory(); + return NULL; + } + } + init_keys_object(dk, log2_size, log2_bytes, + unicode ? DICT_KEYS_UNICODE : DICT_KEYS_GENERAL, + usable, entry_size); return dk; } static void free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { + void *ptr = keys; +#ifdef Py_GIL_DISABLED + size_t size = _PyDict_KeysSize(keys); +#endif + if (keys->dk_kind == DICT_KEYS_SPLIT) { + ptr = _PyDictKeys_AsSharedKeys(keys); +#ifdef Py_GIL_DISABLED + size += offsetof(struct _instancekeysobject, dsk_keys); +#endif + } #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(keys, _PyDict_KeysSize(keys)); + _PyMem_FreeDelayed(ptr, size); return; } #endif @@ -853,7 +890,7 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) _Py_FREELIST_FREE(dictkeys, keys, PyMem_Free); } else { - PyMem_Free(keys); + PyMem_Free(ptr); } } @@ -900,7 +937,7 @@ free_values(PyDictValues *values, bool use_qsbr) static inline PyObject * new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, - int free_values_on_failure) + int free_values_on_failure, int frozendict) { assert(keys != NULL); if (mp == NULL) { @@ -915,6 +952,9 @@ new_dict_impl(PyDictObject *mp, PyDictKeysObject *keys, mp->ma_values = values; mp->ma_used = used; mp->_ma_watcher_tag = 0; + if (frozendict) { + ((PyFrozenDictObject *)mp)->ma_hash = -1; + } ASSERT_CONSISTENT(mp); _PyObject_GC_TRACK(mp); return (PyObject *)mp; @@ -931,7 +971,7 @@ new_dict(PyDictKeysObject *keys, PyDictValues *values, } assert(mp == NULL || Py_IS_TYPE(mp, &PyDict_Type)); - return new_dict_impl(mp, keys, values, used, free_values_on_failure); + return new_dict_impl(mp, keys, values, used, free_values_on_failure, 0); } /* Consumes a reference to the keys object */ @@ -940,7 +980,7 @@ new_frozendict(PyDictKeysObject *keys, PyDictValues *values, Py_ssize_t used, int free_values_on_failure) { PyDictObject *mp = PyObject_GC_New(PyDictObject, &PyFrozenDict_Type); - return new_dict_impl(mp, keys, values, used, free_values_on_failure); + return new_dict_impl(mp, keys, values, used, free_values_on_failure, 1); } static PyObject * @@ -1893,10 +1933,12 @@ insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash) } #endif + bool inserted = false; LOCK_KEYS(keys); ix = unicodekeys_lookup_unicode(keys, key, hash); if (ix == DKIX_EMPTY && keys->dk_usable > 0) { // Insert into new slot + inserted = true; FT_ATOMIC_STORE_UINT32_RELAXED(keys->dk_version, 0); Py_ssize_t hashpos = find_empty_slot(keys, hash); ix = keys->dk_nentries; @@ -1907,6 +1949,13 @@ insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash) } assert (ix < SHARED_KEYS_MAX_SIZE); UNLOCK_KEYS(keys); + + if (inserted) { + // gh-151593: Calling PyType_Modified() with LOCK_KEYS() creates a + // deadlock. So only call the function after UNLOCK_KEYS(). + _PyDict_SplitKeysInvalidated(keys); + } + return ix; } @@ -2354,7 +2403,7 @@ dict_getitem(PyObject *op, PyObject *key, const char *warnmsg) } PyDictObject *mp = (PyDictObject *)op; - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { PyErr_FormatUnraisable(warnmsg); return NULL; @@ -2426,7 +2475,7 @@ _PyDict_LookupIndexAndValue(PyDictObject *mp, PyObject *key, PyObject **value) assert(PyDict_CheckExact((PyObject*)mp)); assert(PyUnicode_CheckExact(key)); - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type((PyObject*)mp, key); return -1; @@ -2530,7 +2579,7 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) return -1; } - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type(op, key); *result = NULL; @@ -2546,7 +2595,7 @@ _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject ** ASSERT_DICT_LOCKED(op); assert(PyUnicode_CheckExact(key)); - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type((PyObject*)op, key); *result = NULL; @@ -2584,7 +2633,7 @@ PyDict_GetItemWithError(PyObject *op, PyObject *key) PyErr_BadInternalCall(); return NULL; } - hash = _PyObject_HashFast(key); + hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type(op, key); return NULL; @@ -2643,7 +2692,7 @@ _PyDict_LoadGlobal(PyDictObject *globals, PyDictObject *builtins, PyObject *key) Py_hash_t hash; PyObject *value; - hash = _PyObject_HashFast(key); + hash = _PyObject_HashDictKey(key); if (hash == -1) { return NULL; } @@ -2667,7 +2716,7 @@ _PyDict_LoadGlobalStackRef(PyDictObject *globals, PyDictObject *builtins, PyObje Py_ssize_t ix; Py_hash_t hash; - hash = _PyObject_HashFast(key); + hash = _PyObject_HashDictKey(key); if (hash == -1) { *res = PyStackRef_NULL; return; @@ -2744,7 +2793,7 @@ setitem_take2_lock_held_known_hash(PyDictObject *mp, PyObject *key, PyObject *va static int setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) { - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type((PyObject*)mp, key); Py_DECREF(key); @@ -2922,7 +2971,7 @@ int PyDict_DelItem(PyObject *op, PyObject *key) { assert(key); - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type(op, key); return -1; @@ -3036,7 +3085,7 @@ clear_embedded_values(PyDictValues *values, Py_ssize_t nentries) assert(nentries <= SHARED_KEYS_MAX_SIZE); for (Py_ssize_t i = 0; i < nentries; i++) { refs[i] = values->values[i]; - values->values[i] = NULL; + FT_ATOMIC_STORE_PTR_RELEASE(values->values[i], NULL); } values->size = 0; for (Py_ssize_t i = 0; i < nentries; i++) { @@ -3080,10 +3129,12 @@ clear_lock_held(PyObject *op) set_keys(mp, Py_EMPTY_KEYS); n = oldkeys->dk_nentries; for (i = 0; i < n; i++) { - Py_CLEAR(oldvalues->values[i]); + PyObject *tmp = oldvalues->values[i]; + FT_ATOMIC_STORE_PTR_RELEASE(oldvalues->values[i], NULL); + Py_XDECREF(tmp); } free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(oldkeys, false); + dictkeys_decref(oldkeys, IS_DICT_SHARED(mp)); } ASSERT_CONSISTENT(mp); } @@ -3264,7 +3315,7 @@ pop_lock_held(PyObject *op, PyObject *key, PyObject **result) return 0; } - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type(op, key); if (result) { @@ -3381,40 +3432,47 @@ dict_set_fromkeys(PyDictObject *mp, PyObject *iterable, PyObject *value) PyObject * _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) { - PyObject *it; /* iter(iterable) */ - PyObject *key; + PyObject *it = NULL; /* iter(iterable) */ PyObject *d; - int status; + int need_copy = 0; - d = _PyObject_CallNoArgs(cls); + if (cls == (PyObject*)&PyFrozenDict_Type) { + // gh-151722: Create a frozendict which is not tracked by the GC. + d = frozendict_new_untracked(&PyFrozenDict_Type); + } + else { + // Dict subclass, or frozendict subclass which overrides + // the constructor. + d = _PyObject_CallNoArgs(cls); + } if (d == NULL) { return NULL; } - // If cls is a dict or frozendict subclass with overridden constructor, - // copy the frozendict. - PyTypeObject *cls_type = _PyType_CAST(cls); - if (PyFrozenDict_Check(d) && cls_type->tp_new != frozendict_new) { - // Subclass-friendly copy - PyObject *copy; - if (PyObject_IsSubclass(cls, (PyObject*)&PyFrozenDict_Type)) { - copy = frozendict_new(cls_type, NULL, NULL); - } - else { - copy = dict_new(cls_type, NULL, NULL); - } + // gh-151722: If cls constructor returns a frozendict which is tracked by + // the GC, create a frozendict copy which is not tracked by the GC. + // + // At the function exit, return cls(fd) where fd is a frozendict. + // + // Untracking the frozendict requires tracking again the frozendict on + // error which is more complicated. It's easier to work on a copy. + if (PyFrozenDict_Check(d) && _PyObject_GC_IS_TRACKED(d)) { + need_copy = 1; + + PyObject *copy = frozendict_new_untracked(&PyFrozenDict_Type); if (copy == NULL) { - Py_DECREF(d); - return NULL; + goto Fail; } if (dict_merge(copy, d, 1, NULL) < 0) { - Py_DECREF(d); Py_DECREF(copy); - return NULL; + goto Fail; } Py_SETREF(d, copy); } - assert(!PyFrozenDict_Check(d) || can_modify_dict((PyDictObject*)d)); + if (PyFrozenDict_Check(d)) { + assert(can_modify_dict((PyDictObject*)d)); + assert(!_PyObject_GC_IS_TRACKED(d)); + } if (PyDict_CheckExact(d)) { if (PyDict_CheckExact(iterable)) { @@ -3423,7 +3481,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_BEGIN_CRITICAL_SECTION2(d, iterable); d = (PyObject *)dict_dict_fromkeys(mp, iterable, value); Py_END_CRITICAL_SECTION2(); - return d; + goto Done; } else if (PyFrozenDict_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; @@ -3431,7 +3489,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_BEGIN_CRITICAL_SECTION(d); d = (PyObject *)dict_dict_fromkeys(mp, iterable, value); Py_END_CRITICAL_SECTION(); - return d; + goto Done; } else if (PyAnySet_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; @@ -3439,7 +3497,7 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_BEGIN_CRITICAL_SECTION2(d, iterable); d = (PyObject *)dict_set_fromkeys(mp, iterable, value); Py_END_CRITICAL_SECTION2(); - return d; + goto Done; } } else if (PyFrozenDict_CheckExact(d)) { @@ -3449,12 +3507,12 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_BEGIN_CRITICAL_SECTION(iterable); d = (PyObject *)dict_dict_fromkeys(mp, iterable, value); Py_END_CRITICAL_SECTION(); - return d; + goto Done; } else if (PyFrozenDict_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; d = (PyObject *)dict_dict_fromkeys(mp, iterable, value); - return d; + goto Done; } else if (PyAnySet_CheckExact(iterable)) { PyDictObject *mp = (PyDictObject *)d; @@ -3462,58 +3520,100 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) Py_BEGIN_CRITICAL_SECTION(iterable); d = (PyObject *)dict_set_fromkeys(mp, iterable, value); Py_END_CRITICAL_SECTION(); - return d; + goto Done; } } it = PyObject_GetIter(iterable); if (it == NULL){ - Py_DECREF(d); - return NULL; + goto Fail; } if (PyDict_CheckExact(d)) { + int status = 0; + Py_BEGIN_CRITICAL_SECTION(d); - while ((key = PyIter_Next(it)) != NULL) { + while (1) { + PyObject *key; + status = PyIter_NextItem(it, &key); + if (status <= 0) { + break; + } + status = setitem_lock_held((PyDictObject *)d, key, value); Py_DECREF(key); if (status < 0) { - assert(PyErr_Occurred()); - goto dict_iter_exit; + break; } } -dict_iter_exit:; Py_END_CRITICAL_SECTION(); + + if (status < 0) { + goto Fail; + } } else if (PyFrozenDict_Check(d)) { - while ((key = PyIter_Next(it)) != NULL) { + while (1) { + PyObject *key; + int status = PyIter_NextItem(it, &key); + if (status < 0) { + goto Fail; + } + if (status == 0) { + break; + } + // setitem_take2_lock_held consumes a reference to key status = setitem_take2_lock_held((PyDictObject *)d, key, Py_NewRef(value)); if (status < 0) { - assert(PyErr_Occurred()); goto Fail; } } } else { - while ((key = PyIter_Next(it)) != NULL) { + while (1) { + PyObject *key; + int status = PyIter_NextItem(it, &key); + if (status < 0) { + goto Fail; + } + if (status == 0) { + break; + } + status = PyObject_SetItem(d, key, value); Py_DECREF(key); - if (status < 0) + if (status < 0) { goto Fail; + } } + } - if (PyErr_Occurred()) - goto Fail; + assert(!PyErr_Occurred()); Py_DECREF(it); - return d; + goto Done; Fail: - Py_DECREF(it); + assert(PyErr_Occurred()); + Py_XDECREF(it); Py_DECREF(d); return NULL; + +Done: + if (d == NULL) { + return NULL; + } + + if (need_copy) { + PyObject *copy = _PyObject_CallOneArg(cls, d); + Py_SETREF(d, copy); + } + else if (!_PyObject_GC_IS_TRACKED(d)) { + _PyObject_GC_TRACK(d); + } + return d; } /* Methods */ @@ -3702,7 +3802,7 @@ _PyDict_SubscriptKnownHash(PyObject *self, PyObject *key, Py_hash_t hash) PyObject * _PyDict_Subscript(PyObject *self, PyObject *key) { - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type(self, key); return NULL; @@ -3887,6 +3987,7 @@ PyDict_Items(PyObject *dict) } /*[clinic input] +@permit_long_summary @classmethod dict.fromkeys iterable: object @@ -3898,7 +3999,7 @@ Create a new dictionary with keys from iterable and values set to value. static PyObject * dict_fromkeys_impl(PyTypeObject *type, PyObject *iterable, PyObject *value) -/*[clinic end generated code: output=8fb98e4b10384999 input=382ba4855d0f74c3]*/ +/*[clinic end generated code: output=8fb98e4b10384999 input=3903715eb48b287e]*/ { return _PyDict_FromKeys((PyObject *)type, iterable, value); } @@ -4113,12 +4214,6 @@ dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override, PyObject ** set_keys(mp, keys); STORE_USED(mp, other->ma_used); ASSERT_CONSISTENT(mp); - - if (_PyObject_GC_IS_TRACKED(other) && !_PyObject_GC_IS_TRACKED(mp)) { - /* Maintain tracking. */ - _PyObject_GC_TRACK(mp); - } - return 0; } } @@ -4285,7 +4380,12 @@ dict_merge_api(PyObject *a, PyObject *b, int override, PyObject **dupkey) } return -1; } - return dict_merge(a, b, override, dupkey); + + int res = dict_merge(a, b, override, dupkey); + if (PyDict_Check(a)) { + assert(_PyObject_GC_IS_TRACKED(a)); + } + return res; } int @@ -4444,10 +4544,15 @@ copy_lock_held(PyObject *o, int as_frozendict) } if (copy == NULL) return NULL; - if (dict_merge(copy, o, 1, NULL) == 0) - return copy; - Py_DECREF(copy); - return NULL; + if (dict_merge(copy, o, 1, NULL) < 0) { + Py_DECREF(copy); + return NULL; + } + + if (PyDict_Check(copy)) { + assert(_PyObject_GC_IS_TRACKED(copy)); + } + return copy; } PyObject * @@ -4653,7 +4758,7 @@ dict_get_impl(PyDictObject *self, PyObject *key, PyObject *default_value) Py_hash_t hash; Py_ssize_t ix; - hash = _PyObject_HashFast(key); + hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type((PyObject*)self, key); return NULL; @@ -4690,7 +4795,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu Py_hash_t hash; Py_ssize_t ix; - hash = _PyObject_HashFast(key); + hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type(d, key); if (result) { @@ -5124,14 +5229,15 @@ static PyMethodDef mapp_methods[] = { DICT_CLEAR_METHODDEF DICT_COPY_METHODDEF DICT___REVERSED___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("dicts are generic over two types, signifying (respectively) the types of their keys and values")}, {NULL, NULL} /* sentinel */ }; static int dict_contains(PyObject *op, PyObject *key) { - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = _PyObject_HashDictKey(key); if (hash == -1) { dict_unhashable_type(op, key); return -1; @@ -5207,15 +5313,14 @@ static PyNumberMethods dict_as_number = { .nb_inplace_or = _PyDict_IOr, }; -static PyObject * -dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +static PyObject* +anydict_new_untracked(PyTypeObject *type) { assert(type != NULL); - assert(type->tp_alloc != NULL); - // dict subclasses must implement the GC protocol + // dict and frozendict subclasses must implement the GC protocol assert(_PyType_IS_GC(type)); - PyObject *self = type->tp_alloc(type, 0); + PyObject *self = _PyType_AllocNoTrack(type, 0); if (self == NULL) { return NULL; } @@ -5228,9 +5333,27 @@ dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) d->ma_keys = Py_EMPTY_KEYS; d->ma_values = NULL; ASSERT_CONSISTENT(d); - if (!_PyObject_GC_IS_TRACKED(d)) { - _PyObject_GC_TRACK(d); + return self; +} + +static PyObject* +dict_new_untracked(PyTypeObject *type) +{ + assert(PyObject_IsSubclass((PyObject*)type, (PyObject*)&PyDict_Type)); + + return anydict_new_untracked(type); +} + +static PyObject * +dict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwds)) +{ + /* tp_new ignores args/kwds; args/kwds are consumed by dict_init (tp_init). */ + PyObject *self = dict_new_untracked(type); + if (self == NULL) { + return NULL; } + assert(!_PyObject_GC_IS_TRACKED(self)); + _PyObject_GC_TRACK(self); return self; } @@ -5289,7 +5412,9 @@ frozendict_vectorcall(PyObject *type, PyObject * const*args, return Py_NewRef(args[0]); } - PyObject *self = frozendict_new(_PyType_CAST(type), NULL, NULL); + /* gh-151722: Keep the frozendict untracked until it is fully built, + so a half-built object is never reachable from another thread (using the gc module). */ + PyObject *self = frozendict_new_untracked(_PyType_CAST(type)); if (self == NULL) { return NULL; } @@ -5309,6 +5434,8 @@ frozendict_vectorcall(PyObject *type, PyObject * const*args, } } } + assert(!_PyObject_GC_IS_TRACKED(self)); + _PyObject_GC_TRACK(self); return self; } @@ -7092,16 +7219,24 @@ dictvalues_reversed(PyObject *self, PyObject *Py_UNUSED(ignored)) PyDictKeysObject * _PyDict_NewKeysForClass(PyHeapTypeObject *cls) { - PyDictKeysObject *keys = new_keys_object(NEXT_LOG2_SHARED_KEYS_MAX_SIZE, 1); - if (keys == NULL) { + int log2_bytes = get_log2_bytes(NEXT_LOG2_SHARED_KEYS_MAX_SIZE); + Py_ssize_t usable = USABLE_FRACTION((size_t)1<<NEXT_LOG2_SHARED_KEYS_MAX_SIZE); + + struct _instancekeysobject *shared_keys = + PyMem_Malloc(sizeof(struct _instancekeysobject) + + ((size_t)1 << log2_bytes) + + sizeof(PyDictUnicodeEntry) * usable); + if (shared_keys == NULL) { PyErr_Clear(); + return NULL; } - else { - assert(keys->dk_nentries == 0); - /* Set to max size+1 as it will shrink by one before each new object */ - keys->dk_usable = SHARED_KEYS_MAX_SIZE; - keys->dk_kind = DICT_KEYS_SPLIT; - } + + shared_keys->dsk_owning_type = (PyTypeObject *)cls; + PyDictKeysObject* keys = &shared_keys->dsk_keys; + init_keys_object(keys, NEXT_LOG2_SHARED_KEYS_MAX_SIZE, log2_bytes, DICT_KEYS_SPLIT, + SHARED_KEYS_MAX_SIZE, sizeof(PyDictUnicodeEntry)); + assert(keys->dk_nentries == 0); + /* Set to max size+1 as it will shrink by one before each new object */ if (cls->ht_type.tp_dict) { PyObject *attrs = PyDict_GetItem(cls->ht_type.tp_dict, &_Py_ID(__static_attributes__)); if (attrs != NULL && PyTuple_Check(attrs)) { @@ -7119,6 +7254,25 @@ _PyDict_NewKeysForClass(PyHeapTypeObject *cls) return keys; } +void +_PyDict_RemoveKeysForClass(PyHeapTypeObject *cls) +{ + struct _instancekeysobject *shared_keys = _PyDictKeys_AsSharedKeys(cls->ht_cached_keys); + FT_ATOMIC_STORE_PTR_RELEASE(shared_keys->dsk_owning_type, NULL); + + _PyDictKeys_DecRef(cls->ht_cached_keys); +} + +void +_PyDict_SplitKeysInvalidated(PyDictKeysObject* keys) +{ + struct _instancekeysobject *shared_keys = _PyDictKeys_AsSharedKeys(keys); + PyTypeObject *type = FT_ATOMIC_LOAD_PTR_ACQUIRE(shared_keys->dsk_owning_type); + if (type) { + PyType_Modified(type); + } +} + void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp) { @@ -7237,7 +7391,7 @@ _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) } if (value == NULL) { - Py_hash_t hash = _PyObject_HashFast(name); + Py_hash_t hash = _PyObject_HashDictKey(name); if (hash == -1) { dict_unhashable_type((PyObject*)dict, name); return -1; @@ -8015,13 +8169,19 @@ validate_watcher_id(PyInterpreterState *interp, int watcher_id) PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); return -1; } - if (!interp->dict_state.watchers[watcher_id]) { + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_RELAXED( + interp->dict_state.watchers[watcher_id]); + if (cb == NULL) { PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); return -1; } return 0; } +// In free-threaded builds, Add/Clear serialize on watcher_mutex and publish +// callbacks with release stores. SendEvent reads them lock-free using +// acquire loads. + int PyDict_Watch(int watcher_id, PyObject* dict) { @@ -8033,7 +8193,8 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, (1LL << watcher_id)); + FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + 1ULL << watcher_id); return 0; } @@ -8048,36 +8209,48 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, ~(1LL << watcher_id)); + FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + ~(1ULL << watcher_id)); return 0; } int PyDict_AddWatcher(PyDict_WatchCallback callback) { + int watcher_id = -1; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); /* Some watchers are reserved for CPython, start at the first available one */ for (int i = FIRST_AVAILABLE_WATCHER; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { - interp->dict_state.watchers[i] = callback; - return i; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[i], callback); + watcher_id = i; + goto done; } } - PyErr_SetString(PyExc_RuntimeError, "no more dict watcher IDs available"); - return -1; +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return watcher_id; } int PyDict_ClearWatcher(int watcher_id) { + int res = 0; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); if (validate_watcher_id(interp, watcher_id)) { - return -1; + res = -1; + goto done; } - interp->dict_state.watchers[watcher_id] = NULL; - return 0; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[watcher_id], NULL); +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return res; } static const char * @@ -8102,7 +8275,8 @@ _PyDict_SendEvent(int watcher_bits, PyInterpreterState *interp = _PyInterpreterState_GET(); for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (watcher_bits & 1) { - PyDict_WatchCallback cb = interp->dict_state.watchers[i]; + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_ACQUIRE( + interp->dict_state.watchers[i]); if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { // We don't want to resurrect the dict by potentially having an // unraisablehook keep a reference to it, so we don't pass the @@ -8172,7 +8346,8 @@ static PyMethodDef frozendict_methods[] = { DICT_FROMKEYS_METHODDEF FROZENDICT_COPY_METHODDEF DICT___REVERSED___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("frozendicts are generic over two types, signifying (respectively) the types of the frozendict's keys and values")}, {"__getnewargs__", frozendict_getnewargs, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; @@ -8205,6 +8380,39 @@ _shuffle_bits(Py_uhash_t h) return ((h ^ 89869747UL) ^ (h << 16)) * 3644798167UL; } +// Compute hash((key, value)). +// Code copied from tuple_hash(). +static Py_hash_t +frozendict_pair_hash(Py_hash_t key_hash, PyObject *value) +{ + assert(key_hash != -1); + + const Py_ssize_t len = 2; + Py_uhash_t acc = _PyTuple_HASH_XXPRIME_5; + + Py_uhash_t lane = key_hash; + acc += lane * _PyTuple_HASH_XXPRIME_2; + acc = _PyTuple_HASH_XXROTATE(acc); + acc *= _PyTuple_HASH_XXPRIME_1; + + lane = PyObject_Hash(value); + if (lane == (Py_uhash_t)-1) { + return -1; + } + acc += lane * _PyTuple_HASH_XXPRIME_2; + acc = _PyTuple_HASH_XXROTATE(acc); + acc *= _PyTuple_HASH_XXPRIME_1; + + /* Add input length, mangled to keep the historical value of hash(()). */ + acc += len ^ (_PyTuple_HASH_XXPRIME_5 ^ 3527539UL); + + if (acc == (Py_uhash_t)-1) { + acc = 1546275796; + } + return acc; +} + + // Code copied from frozenset_hash() static Py_hash_t frozendict_hash(PyObject *op) @@ -8218,20 +8426,15 @@ frozendict_hash(PyObject *op) PyDictObject *mp = _PyAnyDict_CAST(op); Py_uhash_t hash = 0; - PyObject *key, *value; // borrowed refs + PyObject *value; // borrowed ref Py_ssize_t pos = 0; - while (PyDict_Next(op, &pos, &key, &value)) { - Py_hash_t key_hash = PyObject_Hash(key); - if (key_hash == -1) { + Py_hash_t key_hash; + while (_PyDict_Next(op, &pos, NULL, &value, &key_hash)) { + Py_hash_t pair_hash = frozendict_pair_hash(key_hash, value); + if (pair_hash == -1) { return -1; } - hash ^= _shuffle_bits(key_hash); - - Py_hash_t value_hash = PyObject_Hash(value); - if (value_hash == -1) { - return -1; - } - hash ^= _shuffle_bits(value_hash); + hash ^= _shuffle_bits(pair_hash); } /* Factor in the number of active entries */ @@ -8251,17 +8454,29 @@ frozendict_hash(PyObject *op) } +/* Allocate an empty, GC-untracked frozendict; the constructor tracks it once + fully built. */ static PyObject * -frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +frozendict_new_untracked(PyTypeObject *type) { - PyObject *d = dict_new(type, args, kwds); + assert(PyObject_IsSubclass((PyObject*)type, (PyObject*)&PyFrozenDict_Type)); + + PyObject *d = anydict_new_untracked(type); if (d == NULL) { return NULL; } assert(can_modify_dict(_PyAnyDict_CAST(d))); + _PyFrozenDictObject_CAST(d)->ma_hash = -1; + return d; +} - PyFrozenDictObject *self = _PyFrozenDictObject_CAST(d); - self->ma_hash = -1; +static PyObject * +frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *d = frozendict_new_untracked(type); + if (d == NULL) { + return NULL; + } if (args != NULL) { if (dict_update_common(d, args, kwds, "frozendict") < 0) { @@ -8273,6 +8488,8 @@ frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) assert(kwds == NULL); } + assert(!_PyObject_GC_IS_TRACKED(d)); + _PyObject_GC_TRACK(d); return d; } diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 364d508dd01822c..fc53f1bfee8dde4 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -290,7 +290,7 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef enum_methods[] = { {"__reduce__", enum_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("'enumerate' objects are generic over the type of their values")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 5e5e87cd6d7559f..149595e64cec144 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -225,7 +225,7 @@ BaseException___reduce___impl(PyBaseExceptionObject *self) */ /*[clinic input] -@critical_section +@critical_section state BaseException.__setstate__ state: object / @@ -233,7 +233,7 @@ BaseException.__setstate__ static PyObject * BaseException___setstate___impl(PyBaseExceptionObject *self, PyObject *state) -/*[clinic end generated code: output=f3834889950453ab input=5524b61cfe9b9856]*/ +/*[clinic end generated code: output=f3834889950453ab input=f9b1aea70382cdb6]*/ { PyObject *d_key, *d_value; Py_ssize_t i = 0; @@ -1743,7 +1743,8 @@ static PyMemberDef BaseExceptionGroup_members[] = { static PyMethodDef BaseExceptionGroup_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Exception groups are generic over the type of their contained exceptions")}, BASEEXCEPTIONGROUP_DERIVE_METHODDEF BASEEXCEPTIONGROUP_SPLIT_METHODDEF BASEEXCEPTIONGROUP_SUBGROUP_METHODDEF @@ -2139,10 +2140,10 @@ oserror_init(PyOSErrorObject *self, PyObject **p_args, return -1; } else { - self->filename = Py_NewRef(filename); + Py_XSETREF(self->filename, Py_NewRef(filename)); if (filename2 && filename2 != Py_None) { - self->filename2 = Py_NewRef(filename2); + Py_XSETREF(self->filename2, Py_NewRef(filename2)); } if (nargs >= 2 && nargs <= 5) { @@ -2157,10 +2158,10 @@ oserror_init(PyOSErrorObject *self, PyObject **p_args, } } } - self->myerrno = Py_XNewRef(myerrno); - self->strerror = Py_XNewRef(strerror); + Py_XSETREF(self->myerrno, Py_XNewRef(myerrno)); + Py_XSETREF(self->strerror, Py_XNewRef(strerror)); #ifdef MS_WINDOWS - self->winerror = Py_XNewRef(winerror); + Py_XSETREF(self->winerror, Py_XNewRef(winerror)); #endif /* Steals the reference to args */ diff --git a/Objects/floatobject.c b/Objects/floatobject.c index d91468dddded9bf..17e6a729dcd83fc 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1670,7 +1670,6 @@ float___getnewargs___impl(PyObject *self) /*[clinic input] -@permit_long_docstring_body @classmethod float.__getformat__ @@ -1683,13 +1682,13 @@ You probably don't want to use this function. It exists mainly to be used in Python's test suite. This function returns whichever of 'IEEE, big-endian' or 'IEEE, -little-endian' best describes the format of floating-point numbers used by the -C type named by typestr. +little-endian' best describes the format of floating-point numbers +used by the C type named by typestr. [clinic start generated code]*/ static PyObject * float___getformat___impl(PyTypeObject *type, const char *typestr) -/*[clinic end generated code: output=2bfb987228cc9628 input=0ae1ba35d192f704]*/ +/*[clinic end generated code: output=2bfb987228cc9628 input=eb1cf45e9bddab72]*/ { if (strcmp(typestr, "double") != 0 && strcmp(typestr, "float") != 0) { PyErr_SetString(PyExc_ValueError, diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 5ae85c5bca61b94..e7ac59379dcfbcc 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -9,6 +9,7 @@ #include "pycore_function.h" // _PyFunction_FromConstructor() #include "pycore_genobject.h" // _PyGen_GetGeneratorFromFrame() #include "pycore_interpframe.h" // _PyFrame_GetLocalsArray() +#include "pycore_list.h" // _PyList_AppendTakeRef() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_metadata.h" // _PyOpcode_Caches @@ -636,9 +637,7 @@ framelocalsproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) goto error; } - int rc = PyList_Append(items, pair); - Py_DECREF(pair); - if (rc < 0) { + if (_PyList_AppendTakeRef((PyListObject *)items, pair) < 0) { goto error; } } @@ -655,9 +654,7 @@ framelocalsproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) goto error; } - int rc = PyList_Append(items, pair); - Py_DECREF(pair); - if (rc < 0) { + if (_PyList_AppendTakeRef((PyListObject *)items, pair) < 0) { goto error; } } @@ -1117,7 +1114,7 @@ frame_back_get_impl(PyFrameObject *self) /*[clinic end generated code: output=3a84c22a55a63c79 input=9e528570d0e1f44a]*/ { PyObject *res = (PyObject *)PyFrame_GetBack(self); - if (res == NULL) { + if (res == NULL && !PyErr_Occurred()) { Py_RETURN_NONE; } return res; @@ -1890,6 +1887,7 @@ frame_trace_set_impl(PyFrameObject *self, PyObject *value) } /*[clinic input] +@permit_long_summary @critical_section @getter frame.f_generator as frame_generator @@ -1899,7 +1897,7 @@ Return the generator or coroutine associated with this frame, or None. static PyObject * frame_generator_get_impl(PyFrameObject *self) -/*[clinic end generated code: output=97aeb2392562e55b input=00a2bd008b239ab0]*/ +/*[clinic end generated code: output=97aeb2392562e55b input=3ffba57ba10f84be]*/ { if (self->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(self->f_frame); @@ -2404,6 +2402,9 @@ PyFrame_GetBack(PyFrameObject *frame) prev = _PyFrame_GetFirstComplete(prev); if (prev) { back = _PyFrame_GetFrameObject(prev); + if (back == NULL) { + return NULL; + } } } return (PyFrameObject*)Py_XNewRef(back); diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index e3bc8eb2739e3fa..c2083e6fcb77f47 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -412,6 +412,9 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje self); } item = _unpack_args(item); + if (item == NULL) { + return NULL; + } for (Py_ssize_t i = 0; i < nparams; i++) { PyObject *param = PyTuple_GET_ITEM(parameters, i); PyObject *prepare, *tmp; @@ -569,7 +572,8 @@ PyDoc_STRVAR(genericalias__doc__, "--\n\n" "Represent a PEP 585 generic type\n" "\n" -"E.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,)."); +"For example, for t = list[int], t.__origin__ is list and t.__args__\n" +"is (int,)."); static PyObject * ga_getitem(PyObject *self, PyObject *item) diff --git a/Objects/genobject.c b/Objects/genobject.c index d628889afc6dc8f..3cdc06733363d3e 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -168,6 +168,7 @@ gen_clear_frame(PyGenObject *gen) { assert(FT_ATOMIC_LOAD_INT8_RELAXED(gen->gi_frame_state) == FRAME_CLEARED); _PyInterpreterFrame *frame = &gen->gi_iframe; + _PyThreadState_UpdateLastProfiledFrame(_PyThreadState_GET(), frame, frame->previous); frame->previous = NULL; _PyFrame_ClearExceptCode(frame); _PyErr_ClearExcState(&gen->gi_exc_state); @@ -681,6 +682,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, 'yield from' or awaiting on with 'await'. */ ret = _gen_throw((PyGenObject *)yf, close_on_genexit, typ, val, tb); + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); tstate->current_frame = prev; frame->previous = NULL; } @@ -701,6 +703,7 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, frame->previous = prev; tstate->current_frame = frame; ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); tstate->current_frame = prev; frame->previous = NULL; Py_DECREF(meth); @@ -1023,7 +1026,8 @@ static PyMethodDef gen_methods[] = { {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, {"close", gen_close, METH_NOARGS, close_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("generators are generic over the types of their yield, send, and return values")}, {NULL, NULL} /* Sentinel */ }; @@ -1374,7 +1378,8 @@ static PyMethodDef coro_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, {"close", gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("coroutines are generic over the types of their yield, send, and return values")}, {NULL, NULL} /* Sentinel */ }; @@ -1820,7 +1825,7 @@ static PyMethodDef async_gen_methods[] = { {"aclose", async_gen_aclose, METH_NOARGS, async_aclose_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("async generators are generic over the types of their yield and send values")}, {NULL, NULL} /* Sentinel */ }; @@ -2002,6 +2007,19 @@ async_gen_asend_send(PyObject *self, PyObject *arg) return result; } +PySendResult +_PyAsyncGenASend_Send(PyObject *iter, PyObject *arg, PyObject **result) +{ + *result = async_gen_asend_send(iter, arg); + if (*result != NULL) { + return PYGEN_NEXT; + } + if (_PyGen_FetchStopIterationValue(result) == 0) { + return PYGEN_RETURN; + } + return PYGEN_ERROR; +} + static PyObject * async_gen_asend_iternext(PyObject *ags) @@ -2090,10 +2108,8 @@ static PyMethodDef async_gen_asend_methods[] = { static PyAsyncMethods async_gen_asend_as_async = { - PyObject_SelfIter, /* am_await */ - 0, /* am_aiter */ - 0, /* am_anext */ - 0, /* am_send */ + .am_await = PyObject_SelfIter, + .am_send = _PyAsyncGenASend_Send, }; diff --git a/Objects/interpolationobject.c b/Objects/interpolationobject.c index b58adb693f0cae5..e37724fb7852a27 100644 --- a/Objects/interpolationobject.c +++ b/Objects/interpolationobject.c @@ -138,7 +138,7 @@ static PyMethodDef interpolation_methods[] = { {"__reduce__", interpolation_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("Interpolations are generic over the types of their values")}, {NULL, NULL}, }; diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index 451f335e033f16b..fa1eb25047d9617 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -135,7 +135,7 @@ PyDoc_STRVAR(lazy_import_doc, "lazy_import(builtins, name, fromlist=None, /)\n" "--\n" "\n" -"Represents a deferred import that will be resolved on first use.\n" +"Represents a lazy import that will be resolved on first use.\n" "\n" "Instances of this object accessed from the global scope will be\n" "automatically imported based upon their name and then replaced with\n" diff --git a/Objects/listobject.c b/Objects/listobject.c index 10e25bbdcdcb6c5..8a9c9bda68269b8 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -2922,7 +2922,6 @@ unsafe_tuple_compare(PyObject *v, PyObject *w, MergeState *ms) * duplicated). */ /*[clinic input] -@permit_long_docstring_body @critical_section list.sort @@ -2932,18 +2931,18 @@ list.sort Sort the list in ascending order and return None. -The sort is in-place (i.e. the list itself is modified) and stable (i.e. the -order of two equal elements is maintained). +The sort is in-place (i.e. the list itself is modified) and stable +(i.e. the order of two equal elements is maintained). -If a key function is given, apply it once to each list item and sort them, -ascending or descending, according to their function values. +If a key function is given, apply it once to each list item and sort +them, ascending or descending, according to their function values. The reverse flag can be set to sort in descending order. [clinic start generated code]*/ static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=e4f6b6069181ad7d]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=c145526281e1fb9f]*/ { MergeState ms; Py_ssize_t nremaining; @@ -3611,7 +3610,8 @@ static PyMethodDef list_methods[] = { LIST_COUNT_METHODDEF LIST_REVERSE_METHODDEF LIST_SORT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("lists are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; @@ -3793,16 +3793,13 @@ list_ass_subscript_lock_held(PyObject *_self, PyObject *item, PyObject *value) lim = Py_SIZE(self) - cur - 1; } - memmove(self->ob_item + cur - i, - self->ob_item + cur + 1, - lim * sizeof(PyObject *)); + ptr_wise_atomic_memmove(self, self->ob_item + cur - i, + self->ob_item + cur + 1, lim); } cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { - memmove(self->ob_item + cur - slicelength, - self->ob_item + cur, - (Py_SIZE(self) - cur) * - sizeof(PyObject *)); + ptr_wise_atomic_memmove(self, self->ob_item + cur - slicelength, + self->ob_item + cur, Py_SIZE(self) - cur); } Py_SET_SIZE(self, Py_SIZE(self) - slicelength); diff --git a/Objects/longobject.c b/Objects/longobject.c index 549cf0b8f12b4e4..7a38ae8ea5a36f0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2125,7 +2125,7 @@ long_to_decimal_string_internal(PyObject *aa, if (size_a >= 10 * _PY_LONG_MAX_STR_DIGITS_THRESHOLD / (3 * PyLong_SHIFT) + 2) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int max_str_digits = interp->long_state.max_str_digits; + int max_str_digits = _Py_atomic_load_int(&interp->long_state.max_str_digits); if ((max_str_digits > 0) && (max_str_digits / (3 * PyLong_SHIFT) <= (size_a - 11) / 10)) { PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT_TO_STR, @@ -2206,7 +2206,7 @@ long_to_decimal_string_internal(PyObject *aa, } if (strlen > _PY_LONG_MAX_STR_DIGITS_THRESHOLD) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int max_str_digits = interp->long_state.max_str_digits; + int max_str_digits = _Py_atomic_load_int(&interp->long_state.max_str_digits); Py_ssize_t strlen_nosign = strlen - negative; if ((max_str_digits > 0) && (strlen_nosign > max_str_digits)) { Py_DECREF(scratch); @@ -3021,7 +3021,7 @@ long_from_string_base(const char **str, int base, PyLongObject **res) * quadratic algorithm. */ if (digits > _PY_LONG_MAX_STR_DIGITS_THRESHOLD) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int max_str_digits = interp->long_state.max_str_digits; + int max_str_digits = _Py_atomic_load_int(&interp->long_state.max_str_digits); if ((max_str_digits > 0) && (digits > max_str_digits)) { PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT_TO_INT, max_str_digits, digits); @@ -6367,20 +6367,21 @@ int_as_integer_ratio_impl(PyObject *self) int.to_bytes length: Py_ssize_t(allow_negative=False) = 1 - Length of bytes object to use. An OverflowError is raised if the - integer is not representable with the given number of bytes. Default - is length 1. + Length of bytes object to use. An OverflowError is raised if + the integer is not representable with the given number of bytes. + Default is length 1. byteorder: unicode(c_default="NULL") = "big" - The byte order used to represent the integer. If byteorder is 'big', - the most significant byte is at the beginning of the byte array. If - byteorder is 'little', the most significant byte is at the end of the - byte array. To request the native byte order of the host system, use - sys.byteorder as the byte order value. Default is to use 'big'. + The byte order used to represent the integer. If byteorder is + 'big', the most significant byte is at the beginning of the byte + array. If byteorder is 'little', the most significant byte is at + the end of the byte array. To request the native byte order of + the host system, use sys.byteorder as the byte order value. + Default is to use 'big'. * signed as is_signed: bool = False - Determines whether two's complement is used to represent the integer. - If signed is False and a negative integer is given, an OverflowError - is raised. + Determines whether two's complement is used to represent the + integer. If signed is False and a negative integer is given, + an OverflowError is raised. Return an array of bytes representing an integer. [clinic start generated code]*/ @@ -6388,7 +6389,7 @@ Return an array of bytes representing an integer. static PyObject * int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=89c801df114050a3 input=66f9d0c20529b44f]*/ +/*[clinic end generated code: output=89c801df114050a3 input=c74a93c07b2f6526]*/ { int little_endian; if (byteorder == NULL) @@ -6424,18 +6425,20 @@ int.from_bytes bytes as bytes_obj: object Holds the array of bytes to convert. The argument must either - support the buffer protocol or be an iterable object producing bytes. - Bytes and bytearray are examples of built-in objects that support the - buffer protocol. + support the buffer protocol or be an iterable object producing + bytes. Bytes and bytearray are examples of built-in objects that + support the buffer protocol. byteorder: unicode(c_default="NULL") = "big" - The byte order used to represent the integer. If byteorder is 'big', - the most significant byte is at the beginning of the byte array. If - byteorder is 'little', the most significant byte is at the end of the - byte array. To request the native byte order of the host system, use - sys.byteorder as the byte order value. Default is to use 'big'. + The byte order used to represent the integer. If byteorder is + 'big', the most significant byte is at the beginning of the byte + array. If byteorder is 'little', the most significant byte is at + the end of the byte array. To request the native byte order of + the host system, use sys.byteorder as the byte order value. + Default is to use 'big'. * signed as is_signed: bool = False - Indicates whether two's complement is used to represent the integer. + Indicates whether two's complement is used to represent the + integer. Return the integer represented by the given array of bytes. [clinic start generated code]*/ @@ -6443,7 +6446,7 @@ Return the integer represented by the given array of bytes. static PyObject * int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=efc5d68e31f9314f input=2ff527997fe7b0c5]*/ +/*[clinic end generated code: output=efc5d68e31f9314f input=95801e50b942e164]*/ { int little_endian; PyObject *long_obj, *bytes; @@ -6508,6 +6511,7 @@ long_long_getter(PyObject *self, void *Py_UNUSED(ignored)) } /*[clinic input] +@permit_long_summary int.is_integer Returns True. Exists for duck type compatibility with float.is_integer. @@ -6515,7 +6519,7 @@ Returns True. Exists for duck type compatibility with float.is_integer. static PyObject * int_is_integer_impl(PyObject *self) -/*[clinic end generated code: output=90f8e794ce5430ef input=7e41c4d4416e05f2]*/ +/*[clinic end generated code: output=90f8e794ce5430ef input=aacf01a2c81c0244]*/ { Py_RETURN_TRUE; } @@ -6597,7 +6601,8 @@ If x is not a number or if base is given, then x must be a string,\n\ bytes, or bytearray instance representing an integer literal in the\n\ given base. The literal can be preceded by '+' or '-' and be surrounded\n\ by whitespace. The base defaults to 10. Valid bases are 0 and 2-36.\n\ -Base 0 means to interpret the base from the string as an integer literal.\n\ +Base 0 means to interpret the base from the string as an integer\n\ +iteral.\n\ >>> int('0b100', base=0)\n\ 4"); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 2c2f85ebf6acc0f..d92c7daff15dbf3 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1217,8 +1217,6 @@ get_native_fmtchar(const char **result, const char *fmt) case 'f': size = sizeof(float); break; case 'd': size = sizeof(double); break; case 'e': size = sizeof(float) / 2; break; - case 'F': size = 2*sizeof(float); break; - case 'D': size = 2*sizeof(double); break; case '?': size = sizeof(_Bool); break; case 'P': size = sizeof(void *); break; case 'Z': { @@ -1284,8 +1282,6 @@ get_native_fmtstr(const char *fmt) case 'f': RETURN("f"); case 'd': RETURN("d"); case 'e': RETURN("e"); - case 'F': RETURN("F"); - case 'D': RETURN("D"); case 'Z': { switch (fmt[1]) { case 'f': RETURN("Zf"); @@ -1633,11 +1629,7 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, 1); -#else - self->exports++; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, 1); return 0; } @@ -1646,11 +1638,7 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, -1); -#else - self->exports--; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } @@ -1810,7 +1798,7 @@ pylong_as_zu(PyObject *item) dest = x; \ } while (0) -/* Unpack a single item. 'fmt' can be any native format character in struct +/* Unpack a single item. 'fmt' can be any native format in struct module syntax. This function is very sensitive to small changes. With this layout gcc automatically generates a fast jump table. */ static inline PyObject * @@ -1865,16 +1853,6 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) case 'e': d[0] = PyFloat_Unpack2(ptr, endian); goto convert_double; /* complexes */ - case 'F': - d[0] = PyFloat_Unpack4(ptr, endian); - d[1] = PyFloat_Unpack4(ptr + sizeof(float), endian); - goto convert_double_complex; - - case 'D': - d[0] = PyFloat_Unpack8(ptr, endian); - d[1] = PyFloat_Unpack8(ptr + sizeof(double), endian); - goto convert_double_complex; - case 'Z': { switch (fmt[1]) { case 'f': @@ -1940,7 +1918,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) memcpy(ptr, (char *)&x, sizeof x); \ } while (0) -/* Pack a single item. 'fmt' can be any native format character in +/* Pack a single item. 'fmt' can be any native format in struct module syntax. */ static int pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt) @@ -2055,24 +2033,6 @@ pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt break; /* complexes */ - case 'F': case 'D': - c = PyComplex_AsCComplex(item); - if (c.real == -1.0 && PyErr_Occurred()) { - goto err_occurred; - } - CHECK_RELEASED_INT_AGAIN(self); - if (fmt[0] == 'D') { - double x[2] = {c.real, c.imag}; - - memcpy(ptr, &x, sizeof(x)); - } - else { - float x[2] = {(float)c.real, (float)c.imag}; - - memcpy(ptr, &x, sizeof(x)); - } - break; - case 'Z': { switch (fmt[1]) { case 'f': case 'd': @@ -2379,23 +2339,23 @@ memoryview_tolist_impl(PyMemoryViewObject *self) } /*[clinic input] -@permit_long_docstring_body memoryview.tobytes order: str(accept={str, NoneType}, c_default="NULL") = 'C' Return the data in the buffer as a byte string. -Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of the -original array is converted to C or Fortran order. For contiguous views, -'A' returns an exact copy of the physical memory. In particular, in-memory -Fortran order is preserved. For non-contiguous views, the data is converted -to C first. order=None is the same as order='C'. +Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of +the original array is converted to C or Fortran order. For +contiguous views, 'A' returns an exact copy of the physical memory. +In particular, in-memory Fortran order is preserved. For +non-contiguous views, the data is converted to C first. order=None +is the same as order='C'. [clinic start generated code]*/ static PyObject * memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) -/*[clinic end generated code: output=1288b62560a32a23 input=23c9faf372cfdbcc]*/ +/*[clinic end generated code: output=1288b62560a32a23 input=119c70aa91791dc8]*/ { Py_buffer *src = VIEW_ADDR(self); char ord = 'C'; @@ -2436,8 +2396,8 @@ memoryview.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: Py_ssize_t = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Return the data in the buffer as a str of hexadecimal numbers. @@ -2456,7 +2416,7 @@ Return the data in the buffer as a str of hexadecimal numbers. static PyObject * memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, Py_ssize_t bytes_per_sep) -/*[clinic end generated code: output=c9bb00c7a8e86056 input=dc48a56ed3b058ae]*/ +/*[clinic end generated code: output=c9bb00c7a8e86056 input=3f1c5d08906e3b70]*/ { Py_buffer *src = VIEW_ADDR(self); @@ -2466,9 +2426,9 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, // Prevent 'self' from being freed if computing len(sep) mutates 'self' // in _Py_strhex_with_sep(). // See: https://github.com/python/cpython/issues/143195. - self->exports++; + FT_ATOMIC_ADD_SSIZE(self->exports, 1); PyObject *ret = _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); - self->exports--; + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return ret; } @@ -3138,22 +3098,6 @@ unpack_cmp(const char *p, const char *q, const char *fmt, } /* complexes */ - case 'F': - { - float x[2], y[2]; - - memcpy(&x, p, sizeof(x)); - memcpy(&y, q, sizeof(y)); - return (x[0] == y[0]) && (x[1] == y[1]); - } - case 'D': - { - double x[2], y[2]; - - memcpy(&x, p, sizeof(x)); - memcpy(&y, q, sizeof(y)); - return (x[0] == y[0]) && (x[1] == y[1]); - } case 'Z': { switch (fmt[1]) { case 'f': @@ -3411,9 +3355,9 @@ memory_hash(PyObject *_self) if (view->obj != NULL) { // Prevent 'self' from being freed when computing the item's hash. // See https://github.com/python/cpython/issues/142664. - self->exports++; + FT_ATOMIC_ADD_SSIZE(self->exports, 1); Py_hash_t h = PyObject_Hash(view->obj); - self->exports--; + FT_ATOMIC_ADD_SSIZE(self->exports, -1); if (h == -1) { /* Keep the original error message */ return -1; @@ -3635,7 +3579,8 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_INDEX_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("memoryviews are generic over the type of their underlying data")}, {NULL, NULL} }; diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index 81b241063ff40fc..7711c827a58b1ca 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -183,9 +183,9 @@ mi_heap_t* _mi_heap_main_get(void) { // note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size). typedef struct mi_thread_data_s { - mi_heap_t heap; // must come first due to cast in `_mi_heap_done` + mi_heap_t heap; // must come first due to cast in `_mi_heap_done` mi_tld_t tld; - mi_memid_t memid; + mi_memid_t memid; // must come last due to zero'ing } mi_thread_data_t; @@ -231,7 +231,7 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) { } if (td != NULL && !is_zero) { - _mi_memzero_aligned(td, sizeof(*td)); + _mi_memzero_aligned(td, offsetof(mi_thread_data_t,memid)); } return td; } diff --git a/Objects/mimalloc/prim/unix/prim.c b/Objects/mimalloc/prim/unix/prim.c index 1598ebabf0a4da0..6f78eb72789e8e5 100644 --- a/Objects/mimalloc/prim/unix/prim.c +++ b/Objects/mimalloc/prim/unix/prim.c @@ -751,7 +751,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { #elif defined(__ANDROID__) || defined(__DragonFly__) || \ defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \ - defined(__sun) + defined(__sun) || defined(__CYGWIN__) #include <stdlib.h> bool _mi_prim_random_buf(void* buf, size_t buf_len) { diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 19e5134d5cf26b7..f447403ef31b43a 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -14,6 +14,7 @@ #include "pycore_object.h" // _PyType_AllocNoTrack #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_slots.h" // _PySlotIterator_Init #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -396,22 +397,20 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) return (PyObject*)m; } +typedef PyObject *(*createfunc_t)(PyObject *, PyModuleDef*); + static PyObject * -module_from_def_and_spec( - PyModuleDef* def_like, /* not necessarily a valid Python object */ +module_from_slots_and_spec( + const PySlot *slots, PyObject *spec, int module_api_version, PyModuleDef* original_def /* NULL if not defined by a def */) { - PyModuleDef_Slot* cur_slot; - PyObject *(*create)(PyObject *, PyModuleDef*) = NULL; + createfunc_t create = NULL; PyObject *nameobj; PyObject *m = NULL; - int has_multiple_interpreters_slot = 0; - void *multiple_interpreters = (void *)0; - int has_gil_slot = 0; + uint64_t multiple_interpreters = (uint64_t)Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; bool requires_gil = true; - int has_execution_slots = 0; const char *name; int ret; void *token = NULL; @@ -431,142 +430,57 @@ module_from_def_and_spec( goto error; } - if (def_like->m_size < 0) { - PyErr_Format( - PyExc_SystemError, - "module %s: m_size may not be negative for multi-phase initialization", - name); - goto error; + _PySlotIterator it; + + PyModuleDef _dummy_def = {0}; + PyModuleDef *def_like; + if (slots) { + assert(!original_def); + def_like = &_dummy_def; + _PySlotIterator_Init(&it, slots, _PySlot_KIND_MOD); + } + else { + assert(original_def); + def_like = original_def; + _PySlotIterator_InitLegacy(&it, def_like->m_slots, _PySlot_KIND_MOD); } + it.name = name; - bool seen_m_name_slot = false; - bool seen_m_doc_slot = false; - bool seen_m_size_slot = false; - bool seen_m_methods_slot = false; - bool seen_m_traverse_slot = false; - bool seen_m_clear_slot = false; - bool seen_m_free_slot = false; - bool seen_m_abi_slot = false; - for (cur_slot = def_like->m_slots; cur_slot && cur_slot->slot; cur_slot++) { - - // Macro to copy a non-NULL, non-repeatable slot. -#define COPY_NONNULL_SLOT(SLOTNAME, TYPE, DEST) \ - do { \ - if (!(TYPE)(cur_slot->value)) { \ - PyErr_Format( \ - PyExc_SystemError, \ - "module %s: %s must not be NULL", \ - name, SLOTNAME); \ - goto error; \ - } \ - DEST = (TYPE)(cur_slot->value); \ - } while (0); \ - ///////////////////////////////////////////////////////////////// - - // Macro to copy a non-NULL, non-repeatable slot to def_like. -#define COPY_DEF_SLOT(SLOTNAME, TYPE, MEMBER) \ - do { \ - if (seen_ ## MEMBER ## _slot) { \ - PyErr_Format( \ - PyExc_SystemError, \ - "module %s has more than one %s slot", \ - name, SLOTNAME); \ - goto error; \ - } \ - seen_ ## MEMBER ## _slot = true; \ - if (original_def) { \ - TYPE orig_value = (TYPE)original_def->MEMBER; \ - TYPE new_value = (TYPE)cur_slot->value; \ - if (orig_value != new_value) { \ - PyErr_Format( \ - PyExc_SystemError, \ - "module %s: %s conflicts with " \ - "PyModuleDef." #MEMBER, \ - name, SLOTNAME); \ - goto error; \ - } \ - } \ - COPY_NONNULL_SLOT(SLOTNAME, TYPE, (def_like->MEMBER)) \ - } while (0); \ - ///////////////////////////////////////////////////////////////// - - // Macro to copy a non-NULL, non-repeatable slot without a - // corresponding PyModuleDef member. - // DEST must be initially NULL (so we don't need a seen_* flag). -#define COPY_NONDEF_SLOT(SLOTNAME, TYPE, DEST) \ - do { \ - if (DEST) { \ - PyErr_Format( \ - PyExc_SystemError, \ - "module %s has more than one %s slot", \ - name, SLOTNAME); \ - goto error; \ - } \ - COPY_NONNULL_SLOT(SLOTNAME, TYPE, DEST) \ - } while (0); \ - ///////////////////////////////////////////////////////////////// - - // Define the whole common case -#define DEF_SLOT_CASE(SLOT, TYPE, MEMBER) \ - case SLOT: \ - COPY_DEF_SLOT(#SLOT, TYPE, MEMBER); \ - break; \ - ///////////////////////////////////////////////////////////////// - switch (cur_slot->slot) { + while (_PySlotIterator_Next(&it)) { + switch (it.current.sl_id) { + case Py_slot_invalid: + goto error; case Py_mod_create: - if (create) { - PyErr_Format( - PyExc_SystemError, - "module %s has multiple create slots", - name); - goto error; - } - create = cur_slot->value; + create = (createfunc_t)it.current.sl_func; break; case Py_mod_exec: - has_execution_slots = 1; if (!original_def) { - COPY_NONDEF_SLOT("Py_mod_exec", _Py_modexecfunc, m_exec); + if (m_exec) { + PyErr_Format( + PyExc_SystemError, + "module %s has multiple Py_mod_exec slots", + name); + goto error; + } + m_exec = (_Py_modexecfunc)it.current.sl_func; } break; case Py_mod_multiple_interpreters: - if (has_multiple_interpreters_slot) { - PyErr_Format( - PyExc_SystemError, - "module %s has more than one 'multiple interpreters' " - "slots", - name); - goto error; - } - multiple_interpreters = cur_slot->value; - has_multiple_interpreters_slot = 1; + multiple_interpreters = it.current.sl_uint64; break; case Py_mod_gil: - if (has_gil_slot) { - PyErr_Format( - PyExc_SystemError, - "module %s has more than one 'gil' slot", - name); - goto error; + { + uint64_t val = it.current.sl_uint64; + requires_gil = (val != (uint64_t)Py_MOD_GIL_NOT_USED); } - requires_gil = (cur_slot->value != Py_MOD_GIL_NOT_USED); - has_gil_slot = 1; break; case Py_mod_abi: - if (PyABIInfo_Check((PyABIInfo *)cur_slot->value, name) < 0) { + if (PyABIInfo_Check(it.current.sl_ptr, name) < 0) { goto error; } - seen_m_abi_slot = true; break; - DEF_SLOT_CASE(Py_mod_name, char*, m_name) - DEF_SLOT_CASE(Py_mod_doc, char*, m_doc) - DEF_SLOT_CASE(Py_mod_state_size, Py_ssize_t, m_size) - DEF_SLOT_CASE(Py_mod_methods, PyMethodDef*, m_methods) - DEF_SLOT_CASE(Py_mod_state_traverse, traverseproc, m_traverse) - DEF_SLOT_CASE(Py_mod_state_clear, inquiry, m_clear) - DEF_SLOT_CASE(Py_mod_state_free, freefunc, m_free) case Py_mod_token: - if (original_def && original_def != cur_slot->value) { + if (original_def && original_def != it.current.sl_ptr) { PyErr_Format( PyExc_SystemError, "module %s: arbitrary Py_mod_token not " @@ -574,22 +488,40 @@ module_from_def_and_spec( name); goto error; } - COPY_NONDEF_SLOT("Py_mod_token", void*, token); + token = it.current.sl_ptr; break; - default: - assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); - PyErr_Format( - PyExc_SystemError, - "module %s uses unknown slot ID %i", - name, cur_slot->slot); - goto error; - } + // Common case: Copy a PEP 793 slot to def_like +#define DEF_SLOT_CASE(SLOT, TYPE, SL_MEMBER, MEMBER) \ + case SLOT: \ + do { \ + if (original_def) { \ + TYPE orig_value = (TYPE)original_def->MEMBER; \ + TYPE new_value = (TYPE)it.current.SL_MEMBER; \ + if (orig_value != new_value) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "module %s: %s conflicts with " \ + "PyModuleDef." #MEMBER, \ + name, _PySlot_GetName(it.current.sl_id)); \ + goto error; \ + } \ + } \ + (def_like->MEMBER) = (TYPE)it.current.SL_MEMBER; \ + } while (0); \ + break; \ + ///////////////////////////////////////////////////////////////// + DEF_SLOT_CASE(Py_mod_name, char*, sl_ptr, m_name) + DEF_SLOT_CASE(Py_mod_doc, char*, sl_ptr, m_doc) + DEF_SLOT_CASE(Py_mod_state_size, Py_ssize_t, sl_size, m_size) + DEF_SLOT_CASE(Py_mod_methods, PyMethodDef*, sl_ptr, m_methods) + DEF_SLOT_CASE(Py_mod_state_traverse, + traverseproc, sl_func, m_traverse) + DEF_SLOT_CASE(Py_mod_state_clear, inquiry, sl_func, m_clear) + DEF_SLOT_CASE(Py_mod_state_free, freefunc, sl_func, m_free) #undef DEF_SLOT_CASE -#undef COPY_DEF_SLOT -#undef COPY_NONDEF_SLOT -#undef COPY_NONNULL_SLOT + } } - if (!original_def && !seen_m_abi_slot) { + if (!original_def && !_PySlotIterator_SawSlot(&it, Py_mod_abi)) { PyErr_Format( PyExc_SystemError, "module %s does not define Py_mod_abi," @@ -614,19 +546,24 @@ module_from_def_and_spec( } #endif + if (def_like->m_size < 0) { + PyErr_Format( + PyExc_SystemError, + "module %s: m_size may not be negative for multi-phase initialization", + name); + goto error; + } + /* By default, multi-phase init modules are expected to work under multiple interpreters. */ - if (!has_multiple_interpreters_slot) { - multiple_interpreters = Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED; - } - if (multiple_interpreters == Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { + if (multiple_interpreters == (int64_t)(intptr_t)Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED) { if (!_Py_IsMainInterpreter(interp) && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) { goto error; } } - else if (multiple_interpreters != Py_MOD_PER_INTERPRETER_GIL_SUPPORTED + else if (multiple_interpreters != (int64_t)(intptr_t)Py_MOD_PER_INTERPRETER_GIL_SUPPORTED && interp->ceval.own_gil && !_Py_IsMainInterpreter(interp) && _PyImport_CheckSubinterpIncompatibleExtensionAllowed(name) < 0) @@ -688,7 +625,7 @@ module_from_def_and_spec( name); goto error; } - if (has_execution_slots) { + if (_PySlotIterator_SawSlot(&it, Py_mod_exec)) { PyErr_Format( PyExc_SystemError, "module %s specifies execution slots, but did not create " @@ -733,11 +670,11 @@ PyObject * PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_version) { PyModuleDef_Init(def); - return module_from_def_and_spec(def, spec, module_api_version, def); + return module_from_slots_and_spec(NULL, spec, module_api_version, def); } PyObject * -PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) +PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec) { if (!slots) { PyErr_SetString( @@ -745,11 +682,9 @@ PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) "PyModule_FromSlotsAndSpec called with NULL slots"); return NULL; } - // Fill in enough of a PyModuleDef to pass to common machinery - PyModuleDef def_like = {.m_slots = (PyModuleDef_Slot *)slots}; - return module_from_def_and_spec(&def_like, spec, PYTHON_API_VERSION, - NULL); + return module_from_slots_and_spec(slots, spec, PYTHON_API_VERSION, + NULL); } #ifdef Py_GIL_DISABLED @@ -835,8 +770,6 @@ PyModule_Exec(PyObject *module) int PyModule_ExecDef(PyObject *module, PyModuleDef *def) { - PyModuleDef_Slot *cur_slot; - if (alloc_state(module) < 0) { return -1; } @@ -847,13 +780,19 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) return 0; } - for (cur_slot = def->m_slots; cur_slot && cur_slot->slot; cur_slot++) { - if (cur_slot->slot == Py_mod_exec) { - int (*func)(PyObject *) = cur_slot->value; - if (run_exec_func(module, func) < 0) { + _PySlotIterator it; + _PySlotIterator_InitLegacy(&it, def->m_slots, _PySlot_KIND_MOD); + while (_PySlotIterator_Next(&it)) { + _Py_modexecfunc func; + switch (it.current.sl_id) { + case Py_slot_invalid: return -1; - } - continue; + case Py_mod_exec: + func = (_Py_modexecfunc)it.current.sl_func; + if (run_exec_func(module, func) < 0) { + return -1; + } + break; } } return 0; @@ -1360,6 +1299,33 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } +// Check if `name` is a lazily pending submodule of module `m`. +// Returns a new reference on success, or NULL with no error set. +static PyObject * +try_load_lazy_submodule(PyModuleObject *m, PyObject *name) +{ + PyObject *mod_name; + int rc = PyDict_GetItemRef(m->md_dict, &_Py_ID(__name__), &mod_name); + if (rc <= 0) { + return NULL; + } + if (!PyUnicode_Check(mod_name)) { + Py_DECREF(mod_name); + return NULL; + } + PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name); + Py_DECREF(mod_name); + if (result == NULL) { + PyErr_Clear(); + return NULL; + } + if (PyDict_SetItem(m->md_dict, name, result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { @@ -1368,6 +1334,25 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); if (attr) { if (PyLazyImport_CheckExact(attr)) { + // gh-144957: Module __getattr__ should get a chance to provide + // the attribute before resolving a lazy import placeholder. + if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { + Py_DECREF(attr); + return NULL; + } + if (getattr) { + PyObject *result = PyObject_CallOneArg(getattr, name); + Py_DECREF(getattr); + if (result != NULL) { + Py_DECREF(attr); + return result; + } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + Py_DECREF(attr); + return NULL; + } + PyErr_Clear(); + } PyObject *new_value = _PyImport_LoadLazyImportTstate( PyThreadState_GET(), attr); if (new_value == NULL) { @@ -1405,6 +1390,13 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) PyErr_Clear(); } assert(m->md_dict != NULL); + attr = try_load_lazy_submodule(m, name); + if (attr != NULL) { + return attr; + } + if (PyErr_Occurred()) { + return NULL; + } if (PyDict_GetItemRef(m->md_dict, &_Py_ID(__getattr__), &getattr) < 0) { return NULL; } diff --git a/Objects/object.c b/Objects/object.c index e0e26bb50d36537..bd23c2e23881949 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2821,6 +2821,13 @@ PyUnstable_Object_EnableDeferredRefcount(PyObject *op) return 0; } + if (!PyObject_GC_IsTracked(op)) { + // When deferred refcount is enabled, the object will only be + // deallocated by the tracing garbage collector. So it must be tracked + // by the garbage collector. + return 0; + } + uint8_t bits = _Py_atomic_load_uint8(&op->ob_gc_bits); if ((bits & _PyGC_BITS_DEFERRED) != 0) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 1809bd30451327b..0947d47c8a55582 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -363,7 +363,10 @@ _PyObject_MiRealloc(void *ctx, void *ptr, size_t nbytes) _mi_memcpy((char*)newp + offset, (char*)ptr + offset, copy_size - offset); } else { - _mi_memcpy(newp, ptr, copy_size); + // memcpy(dst, NULL, 0) is undefined behavior. See gh-151297. + if mi_likely(ptr) { + _mi_memcpy(newp, ptr, copy_size); + } } mi_free(ptr); return newp; diff --git a/Objects/odictobject.c b/Objects/odictobject.c index b391283e83795d0..14c1b02405822cd 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -1149,12 +1149,13 @@ OrderedDict.popitem Remove and return a (key, value) pair from the dictionary. -Pairs are returned in LIFO order if last is true or FIFO order if false. +Pairs are returned in LIFO order if last is true or FIFO order if +false. [clinic start generated code]*/ static PyObject * OrderedDict_popitem_impl(PyODictObject *self, int last) -/*[clinic end generated code: output=98e7d986690d49eb input=8aafc7433e0a40e7]*/ +/*[clinic end generated code: output=98e7d986690d49eb input=ebf1cc91579c9e54]*/ { PyObject *key, *value; _ODictNode *node; @@ -2205,13 +2206,14 @@ update __setitem__ static int mutablemapping_add_pairs(PyObject *self, PyObject *pairs) { + assert(!PyErr_Occurred()); + PyObject *pair, *iterator, *unexpected; int res = 0; iterator = PyObject_GetIter(pairs); if (iterator == NULL) return -1; - PyErr_Clear(); while ((pair = PyIter_Next(iterator)) != NULL) { /* could be more efficient (see UNPACK_SEQUENCE in ceval.c) */ diff --git a/Objects/sentinelobject.c b/Objects/sentinelobject.c index e7e9f60e3edfbe9..77bffbc397be585 100644 --- a/Objects/sentinelobject.c +++ b/Objects/sentinelobject.c @@ -14,6 +14,7 @@ typedef struct { PyObject_HEAD PyObject *name; PyObject *module; + PyObject *repr; } sentinelobject; #define sentinelobject_CAST(op) \ @@ -46,7 +47,7 @@ caller(void) } static PyObject * -sentinel_new_with_module(PyTypeObject *type, PyObject *name, PyObject *module) +sentinel_new_with_module(PyTypeObject *type, PyObject *name, PyObject *module, PyObject *repr) { assert(PyUnicode_Check(name)); @@ -56,6 +57,7 @@ sentinel_new_with_module(PyTypeObject *type, PyObject *name, PyObject *module) } self->name = Py_NewRef(name); self->module = Py_NewRef(module); + self->repr = Py_XNewRef(repr); _PyObject_GC_TRACK(self); return (PyObject *)self; } @@ -66,37 +68,56 @@ sentinel.__new__ as sentinel_new name: object(subclass_of='&PyUnicode_Type') / + * + repr: object = None [clinic start generated code]*/ static PyObject * -sentinel_new_impl(PyTypeObject *type, PyObject *name) -/*[clinic end generated code: output=4af55c6048bed30d input=3ab75704f39c119c]*/ +sentinel_new_impl(PyTypeObject *type, PyObject *name, PyObject *repr) +/*[clinic end generated code: output=1eb7fab52e57d8c8 input=28cab6c468997b35]*/ { + if (repr == Py_None) { + repr = NULL; + } + else if (!PyUnicode_Check(repr)) { + _PyArg_BadArgument("sentinel", "argument 'repr'", "str or None", repr); + return NULL; + } PyObject *module = caller(); - PyObject *self = sentinel_new_with_module(type, name, module); + PyObject *self = sentinel_new_with_module(type, name, module, repr); Py_DECREF(module); return self; } PyObject * -PySentinel_New(const char *name, const char *module_name) +PySentinel_New(const char *name, const char *module_name, const char *repr) { PyObject *name_obj = PyUnicode_FromString(name); if (name_obj == NULL) { return NULL; } + PyObject *repr_obj = NULL; + if (repr != NULL) { + repr_obj = PyUnicode_FromString(repr); + if (repr_obj == NULL) { + Py_DECREF(name_obj); + return NULL; + } + } PyObject *module_obj = module_name == NULL ? Py_None : PyUnicode_FromString(module_name); if (module_obj == NULL) { Py_DECREF(name_obj); + Py_XDECREF(repr_obj); return NULL; } PyObject *sentinel = sentinel_new_with_module( - &PySentinel_Type, name_obj, module_obj); + &PySentinel_Type, name_obj, module_obj, repr_obj); Py_DECREF(module_obj); Py_DECREF(name_obj); + Py_XDECREF(repr_obj); return sentinel; } @@ -106,6 +127,7 @@ sentinel_clear(PyObject *op) sentinelobject *self = sentinelobject_CAST(op); Py_CLEAR(self->name); Py_CLEAR(self->module); + Py_CLEAR(self->repr); return 0; } @@ -123,6 +145,7 @@ sentinel_traverse(PyObject *op, visitproc visit, void *arg) sentinelobject *self = sentinelobject_CAST(op); Py_VISIT(self->name); Py_VISIT(self->module); + Py_VISIT(self->repr); return 0; } @@ -130,6 +153,9 @@ static PyObject * sentinel_repr(PyObject *op) { sentinelobject *self = sentinelobject_CAST(op); + if (self->repr != NULL) { + return Py_NewRef(self->repr); + } return Py_NewRef(self->name); } @@ -161,7 +187,7 @@ static PyMethodDef sentinel_methods[] = { static PyMemberDef sentinel_members[] = { {"__name__", Py_T_OBJECT_EX, offsetof(sentinelobject, name), Py_READONLY}, - {"__module__", Py_T_OBJECT_EX, offsetof(sentinelobject, module), Py_READONLY}, + {"__module__", Py_T_OBJECT_EX, offsetof(sentinelobject, module), 0}, {NULL} }; @@ -170,7 +196,7 @@ static PyNumberMethods sentinel_as_number = { }; PyDoc_STRVAR(sentinel_doc, -"sentinel(name, /)\n" +"sentinel(name, /, *, repr=None)\n" "--\n\n" "Create a unique sentinel object with the given name."); diff --git a/Objects/setobject.c b/Objects/setobject.c index 1e6305636045529..883658b8bfd340d 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -361,7 +361,7 @@ set_unhashable_type(PyObject *key) int _PySet_AddTakeRef(PySetObject *so, PyObject *key) { - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { set_unhashable_type(key); Py_DECREF(key); @@ -600,7 +600,7 @@ set_discard_entry(PySetObject *so, PyObject *key, Py_hash_t hash) static int set_add_key(PySetObject *so, PyObject *key) { - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { set_unhashable_type(key); return -1; @@ -611,7 +611,7 @@ set_add_key(PySetObject *so, PyObject *key) static int set_contains_key(PySetObject *so, PyObject *key) { - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { set_unhashable_type(key); return -1; @@ -622,7 +622,7 @@ set_contains_key(PySetObject *so, PyObject *key) static int set_discard_key(PySetObject *so, PyObject *key) { - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { set_unhashable_type(key); return -1; @@ -1351,7 +1351,9 @@ make_new_set(PyTypeObject *type, PyObject *iterable) assert(PyType_Check(type)); PySetObject *so; - so = (PySetObject *)type->tp_alloc(type, 0); + // Allocate untracked: the fill below runs user code, and a half-built + // set must not be reachable from another thread via gc.get_objects(). + so = (PySetObject *)_PyType_AllocNoTrack(type, 0); if (so == NULL) return NULL; @@ -1370,6 +1372,8 @@ make_new_set(PyTypeObject *type, PyObject *iterable) } } + // Track only once fully built. + _PyObject_GC_TRACK(so); return (PyObject *)so; } @@ -1545,6 +1549,16 @@ set_swap_bodies(PySetObject *a, PySetObject *b) FT_ATOMIC_STORE_PTR_RELEASE(b->table, b_table); } +PyObject * +_PySet_Freeze(PyObject *set) +{ + assert(set != NULL); + assert(PySet_CheckExact(set)); + assert(_PyObject_IsUniquelyReferenced(set)); + set->ob_type = &PyFrozenSet_Type; + return Py_NewRef(set); +} + /*[clinic input] @critical_section set.copy @@ -2111,6 +2125,7 @@ set_difference(PySetObject *so, PyObject *other) } /*[clinic input] +@permit_long_summary set.difference as set_difference_multi so: setobject *others: array @@ -2121,7 +2136,7 @@ Return a new set with elements in the set that are not in the others. static PyObject * set_difference_multi_impl(PySetObject *so, PyObject * const *others, Py_ssize_t others_length) -/*[clinic end generated code: output=b0d33fb05d5477a7 input=c1eb448d483416ad]*/ +/*[clinic end generated code: output=b0d33fb05d5477a7 input=e0fbedbf79d91d4e]*/ { Py_ssize_t i; PyObject *result, *other; @@ -2293,6 +2308,7 @@ set_symmetric_difference_update_impl(PySetObject *so, PyObject *other) } /*[clinic input] +@permit_long_summary @critical_section so other set.symmetric_difference so: setobject @@ -2304,7 +2320,7 @@ Return a new set with elements in either the set or other but not both. static PyObject * set_symmetric_difference_impl(PySetObject *so, PyObject *other) -/*[clinic end generated code: output=270ee0b5d42b0797 input=624f6e7bbdf70db1]*/ +/*[clinic end generated code: output=270ee0b5d42b0797 input=8c29b0be90d47feb]*/ { PySetObject *result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); if (result == NULL) { @@ -2502,7 +2518,7 @@ _PySet_Contains(PySetObject *so, PyObject *key) { assert(so); - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) { set_unhashable_type(key); @@ -2562,7 +2578,7 @@ static PyObject * frozenset___contains___impl(PySetObject *so, PyObject *key) /*[clinic end generated code: output=2301ed91bc3a6dd5 input=2f04922a98d8bab7]*/ { - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { if (!PySet_Check(key) || !PyErr_ExceptionMatches(PyExc_TypeError)) { set_unhashable_type(key); @@ -2790,7 +2806,8 @@ static PyMethodDef set_methods[] = { SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF SET_UNION_METHODDEF SET_UPDATE_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("sets are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2872,7 +2889,7 @@ PyTypeObject PySet_Type = { 0, /* tp_descr_set */ 0, /* tp_dictoffset */ set_init, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ + _PyType_AllocNoTrack, /* tp_alloc */ set_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = set_vectorcall, @@ -2894,7 +2911,8 @@ static PyMethodDef frozenset_methods[] = { SET___SIZEOF___METHODDEF SET_SYMMETRIC_DIFFERENCE_METHODDEF SET_UNION_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("frozensets are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2963,7 +2981,7 @@ PyTypeObject PyFrozenSet_Type = { 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ + _PyType_AllocNoTrack, /* tp_alloc */ frozenset_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = frozenset_vectorcall, @@ -3025,7 +3043,7 @@ PySet_Contains(PyObject *anyset, PyObject *key) } PySetObject *so = (PySetObject *)anyset; - Py_hash_t hash = _PyObject_HashFast(key); + Py_hash_t hash = PyObject_Hash(key); if (hash == -1) { set_unhashable_type(key); return -1; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 95f10815687757e..91d1753b822fc13 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -117,8 +117,8 @@ PyObject _Py_EllipsisObject = _PyObject_HEAD_INIT(&PyEllipsis_Type); index is present. */ -static PySliceObject * -_PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) +PyObject * +_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop, PyObject *step) { assert(start != NULL && stop != NULL && step != NULL); PySliceObject *obj = _Py_FREELIST_POP(PySliceObject, slices); @@ -131,13 +131,14 @@ _PyBuildSlice_Consume2(PyObject *start, PyObject *stop, PyObject *step) obj->start = start; obj->stop = stop; - obj->step = Py_NewRef(step); + obj->step = step; _PyObject_GC_TRACK(obj); - return obj; + return (PyObject *)obj; error: Py_DECREF(start); Py_DECREF(stop); + Py_DECREF(step); return NULL; } @@ -153,15 +154,8 @@ PySlice_New(PyObject *start, PyObject *stop, PyObject *step) if (stop == NULL) { stop = Py_None; } - return (PyObject *)_PyBuildSlice_Consume2(Py_NewRef(start), - Py_NewRef(stop), step); -} - -PyObject * -_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop) -{ - assert(start != NULL && stop != NULL); - return (PyObject *)_PyBuildSlice_Consume2(start, stop, Py_None); + return _PyBuildSlice_ConsumeRefs(Py_NewRef(start), + Py_NewRef(stop), Py_NewRef(step)); } PyObject * @@ -177,9 +171,7 @@ _PySlice_FromIndices(Py_ssize_t istart, Py_ssize_t istop) return NULL; } - slice = PySlice_New(start, end, NULL); - Py_DECREF(start); - Py_DECREF(end); + slice = _PyBuildSlice_ConsumeRefs(start, end, Py_None); return slice; } @@ -342,7 +334,9 @@ PyDoc_STRVAR(slice_doc, "slice(stop)\n\ slice(start, stop[, step])\n\ \n\ -Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); +Create a slice object.\n\ +\n\ +This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PyObject *op) @@ -570,7 +564,8 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef slice_methods[] = { {"indices", slice_indices, METH_O, slice_indices_doc}, {"__reduce__", slice_reduce, METH_NOARGS, reduce_doc}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + "slices are generic over the types of their start, end, and step values"}, {NULL, NULL} }; diff --git a/Objects/stringlib/join.h b/Objects/stringlib/join.h index de6bd83ffe4c8bc..fe460d41f6700f8 100644 --- a/Objects/stringlib/join.h +++ b/Objects/stringlib/join.h @@ -68,13 +68,18 @@ STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) buffers[i].len = PyBytes_GET_SIZE(item); } else { + /* item is only borrowed; its __buffer__() may run Python that + drops the sequence's last reference to it. */ + Py_INCREF(item); if (PyObject_GetBuffer(item, &buffers[i], PyBUF_SIMPLE) != 0) { PyErr_Format(PyExc_TypeError, "sequence item %zd: expected a bytes-like object, " "%.80s found", i, Py_TYPE(item)->tp_name); + Py_DECREF(item); goto error; } + Py_DECREF(item); /* If the backing objects are mutable, then dropping the GIL * opens up race conditions where another thread tries to modify * the object which we hold a buffer on it. Such code has data diff --git a/Objects/templateobject.c b/Objects/templateobject.c index a05208e4c8fc8e8..1609e82b444516c 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -372,7 +372,11 @@ template_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef template_methods[] = { {"__reduce__", template_reduce, METH_NOARGS, NULL}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + // note that this is not supported in typeshed, and it is not clear if the + // type for this is a simple TypeVar or a TypeVarTuple + // for details, see: https://github.com/python/typeshed/issues/14878 + PyDoc_STR("Template supports [] for generic usage")}, {NULL, NULL}, }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 753c270f525976f..bb5e18cb790acf0 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -363,6 +363,9 @@ tuple_repr(PyObject *self) https://github.com/Cyan4973/xxHash/blob/master/doc/xxhash_spec.md The constants for the hash function are defined in pycore_tuple.h. + + If you update this code, update also frozendict_pair_hash() which copied + this code. */ static Py_hash_t @@ -594,8 +597,8 @@ _PyTuple_Concat(PyObject *aa, PyObject *bb) return (PyObject *)np; } -static PyObject * -tuple_repeat(PyObject *self, Py_ssize_t n) +PyObject * +_PyTuple_Repeat(PyObject *self, Py_ssize_t n) { PyTupleObject *a = _PyTuple_CAST(self); const Py_ssize_t input_size = Py_SIZE(a); @@ -865,7 +868,7 @@ tuple_subtype_new(PyTypeObject *type, PyObject *iterable) static PySequenceMethods tuple_as_sequence = { tuple_length, /* sq_length */ _PyTuple_Concat, /* sq_concat */ - tuple_repeat, /* sq_repeat */ + _PyTuple_Repeat, /* sq_repeat */ tuple_item, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ @@ -951,11 +954,19 @@ tuple___getnewargs___impl(PyTupleObject *self) return Py_BuildValue("(N)", tuple_slice(self, 0, Py_SIZE(self))); } + +PyDoc_STRVAR(tuple_class_getitem_doc, +"Tuples are generic over the types of their contents.\n\n\ +For example, use ``tuple[int, str]`` for a pair whose first element\n\ +is an int and second element is a string.\n\n\ +Tuples also support the form ``tuple[T, ...]`` to indicate\n\ +an arbitrary length tuple of elements of type T."); + static PyMethodDef tuple_methods[] = { TUPLE___GETNEWARGS___METHODDEF TUPLE_INDEX_METHODDEF TUPLE_COUNT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, tuple_class_getitem_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index fb3c71014106835..777cf842fb9602c 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -18,6 +18,7 @@ #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_slots.h" // _PySlotIterator_Init #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_typeobject.h" // struct type_cache @@ -198,11 +199,6 @@ type_lock_allow_release(void) #define PyTypeObject_CAST(op) ((PyTypeObject *)(op)) -typedef struct PySlot_Offset { - short subslot_offset; - short slot_offset; -} PySlot_Offset; - static void slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer); @@ -1598,7 +1594,12 @@ type_set_qualname(PyObject *tp, PyObject *value, void *context) } et = (PyHeapTypeObject*)type; - Py_SETREF(et->ht_qualname, Py_NewRef(value)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + PyObject *old_qualname = et->ht_qualname; + et->ht_qualname = Py_NewRef(value); + _PyEval_StartTheWorld(interp); + Py_DECREF(old_qualname); return 0; } @@ -3716,9 +3717,9 @@ find_best_base(PyObject *bases) for (i = 0; i < n; i++) { PyObject *base_proto = PyTuple_GET_ITEM(bases, i); if (!PyType_Check(base_proto)) { - PyErr_SetString( + PyErr_Format( PyExc_TypeError, - "bases must be types"); + "bases must be types; got '%T'", base_proto); return NULL; } PyTypeObject *base_i = (PyTypeObject *)base_proto; @@ -3792,7 +3793,7 @@ solid_base(PyTypeObject *type) // or when __bases__ is re-assigned. Since the slots are read without atomic // operations and without locking, we can only safely update them while the // world is stopped. However, with the world stopped, we are very limited on -// which APIs can be safely used. For example, calling _PyObject_HashFast() +// which APIs can be safely used. For example, calling _PyObject_HashDictKey() // or _PyDict_GetItemRef_KnownHash() are not safe and can potentially cause // deadlocks. Hashing can be re-entrant and _PyDict_GetItemRef_KnownHash can // acquire a lock if the dictionary is not owned by the current thread, to @@ -4166,8 +4167,9 @@ _PyType_CalculateMetaclass(PyTypeObject *metatype, PyObject *bases) for (i = 0; i < nbases; i++) { tmp = PyTuple_GET_ITEM(bases, i); tmptype = Py_TYPE(tmp); - if (PyType_IsSubtype(winner, tmptype)) + if (PyType_IsSubtype(winner, tmptype)) { continue; + } if (PyType_IsSubtype(tmptype, winner)) { winner = tmptype; continue; @@ -4845,6 +4847,18 @@ type_new_set_attrs(const type_new_ctx *ctx, PyTypeObject *type) if (type_new_set_classdictcell(dict) < 0) { return -1; } + +#ifdef Py_GIL_DISABLED + // enable deferred reference counting on functions and descriptors + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(dict, &pos, &key, &value)) { + if (PyFunction_Check(value) || Py_TYPE(value)->tp_descr_get != NULL) { + PyUnstable_Object_EnableDeferredRefcount(value); + } + } +#endif + return 0; } @@ -5112,21 +5126,6 @@ type_vectorcall(PyObject *metatype, PyObject *const *args, return _PyObject_MakeTpCall(tstate, metatype, args, nargs, kwnames); } -/* An array of type slot offsets corresponding to Py_tp_* constants, - * for use in e.g. PyType_Spec and PyType_GetSlot. - * Each entry has two offsets: "slot_offset" and "subslot_offset". - * If is subslot_offset is -1, slot_offset is an offset within the - * PyTypeObject struct. - * Otherwise slot_offset is an offset to a pointer to a sub-slots struct - * (such as "tp_as_number"), and subslot_offset is the offset within - * that struct. - * The actual table is generated by a script. - */ -static const PySlot_Offset pyslot_offsets[] = { - {0, 0}, -#include "typeslots.inc" -}; - /* Align up to the nearest multiple of alignof(max_align_t) * (like _Py_ALIGN_UP, but for a size rather than pointer) */ @@ -5136,43 +5135,6 @@ _align_up(Py_ssize_t size) return (size + ALIGNOF_MAX_ALIGN_T - 1) & ~(ALIGNOF_MAX_ALIGN_T - 1); } -/* Given a PyType_FromMetaclass `bases` argument (NULL, type, or tuple of - * types), return a tuple of types. - */ -inline static PyObject * -get_bases_tuple(PyObject *bases_in, PyType_Spec *spec) -{ - if (!bases_in) { - /* Default: look in the spec, fall back to (type,). */ - PyTypeObject *base = &PyBaseObject_Type; // borrowed ref - PyObject *bases = NULL; // borrowed ref - const PyType_Slot *slot; - for (slot = spec->slots; slot->slot; slot++) { - switch (slot->slot) { - case Py_tp_base: - base = slot->pfunc; - break; - case Py_tp_bases: - bases = slot->pfunc; - break; - } - } - if (!bases) { - return PyTuple_Pack(1, base); - } - if (PyTuple_Check(bases)) { - return Py_NewRef(bases); - } - PyErr_SetString(PyExc_SystemError, "Py_tp_bases is not a tuple"); - return NULL; - } - if (PyTuple_Check(bases_in)) { - return Py_NewRef(bases_in); - } - // Not a tuple, should be a single type - return PyTuple_Pack(1, bases_in); -} - static inline int check_basicsize_includes_size_and_offsets(PyTypeObject* type) { @@ -5274,10 +5236,11 @@ special_offset_from_member( return -1; } -PyObject * -PyType_FromMetaclass( - PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) + +static PyObject * +type_from_slots_or_spec( + PySlot *slots, PyType_Spec *spec, + PyTypeObject *metaclass, PyObject *module, PyObject *bases_in) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -5292,47 +5255,130 @@ PyType_FromMetaclass( int r; - /* Prepare slots that need special handling. - * Keep in mind that a slot can be given multiple times: - * if that would cause trouble (leaks, UB, ...), raise an exception. - */ + /* First pass of slots */ - const PyType_Slot *slot; Py_ssize_t nmembers = 0; const PyMemberDef *weaklistoffset_member = NULL; const PyMemberDef *dictoffset_member = NULL; const PyMemberDef *vectorcalloffset_member = NULL; - char *res_start; + Py_ssize_t basicsize = 0; + Py_ssize_t extra_basicsize = 0; + Py_ssize_t itemsize = 0; + int flags = 0; + void *token = NULL; - for (slot = spec->slots; slot->slot; slot++) { - if (slot->slot < 0 - || (size_t)slot->slot >= Py_ARRAY_LENGTH(pyslot_offsets)) { - PyErr_SetString(PyExc_RuntimeError, "invalid slot offset"); - goto finally; + bool have_relative_members = false; + Py_ssize_t max_relative_offset = 0; + + PyObject *bases_slot = NULL; /* borrowed from the slots */ + + _PySlotIterator it; + + if (spec) { + assert(!slots); + if (spec->basicsize > 0) { + basicsize = spec->basicsize; } - switch (slot->slot) { - case Py_tp_members: - if (nmembers != 0) { + if (spec->basicsize < 0) { + extra_basicsize = -spec->basicsize; + } + itemsize = spec->itemsize; + flags = spec->flags; + _PySlotIterator_InitLegacy(&it, spec->slots, _PySlot_KIND_TYPE); + it.name = spec->name; + } + else { + assert(!spec); + assert(!metaclass); + assert(!module); + assert(!bases_in); + _PySlotIterator_Init(&it, slots, _PySlot_KIND_TYPE); + } + + #define NO_SPEC \ + if (spec) { \ + PyErr_Format( \ + PyExc_SystemError, \ + "%s must not be used with PyType_Spec", \ + _PySlot_GetName(it.current.sl_id)); \ + goto finally; \ + } \ + ///////////////////////////////////////////////////// + + while (_PySlotIterator_Next(&it)) { + switch (it.current.sl_id) { + case Py_slot_invalid: + goto finally; + case Py_tp_name: + NO_SPEC; + it.name = it.current.sl_ptr; + break; + case Py_tp_metaclass: + NO_SPEC; + metaclass = it.current.sl_ptr; + break; + case Py_tp_module: + NO_SPEC; + module = it.current.sl_ptr; + break; + case Py_tp_bases: + bases_slot = it.current.sl_ptr; + break; + case Py_tp_base: + if (!_PySlotIterator_SawSlot(&it, Py_tp_bases)) { + bases_slot = it.current.sl_ptr; + } + break; + case Py_tp_basicsize: + NO_SPEC; + basicsize = it.current.sl_size; + if (basicsize <= 0) { + PyErr_SetString( + PyExc_SystemError, + "Py_tp_basicsize must be positive"); + goto finally; + } + break; + case Py_tp_extra_basicsize: + NO_SPEC; + extra_basicsize = it.current.sl_size; + if (extra_basicsize <= 0) { PyErr_SetString( PyExc_SystemError, - "Multiple Py_tp_members slots are not supported."); + "Py_tp_extra_basicsize must be positive"); goto finally; } - for (const PyMemberDef *memb = slot->pfunc; memb->name != NULL; memb++) { + break; + case Py_tp_itemsize: + NO_SPEC; + itemsize = it.current.sl_size; + if (itemsize <= 0) { + PyErr_SetString( + PyExc_SystemError, + "Py_tp_itemsize must be positive"); + goto finally; + } + break; + case Py_tp_flags: + NO_SPEC; + flags = (int)it.current.sl_uint64; + break; + case Py_tp_members: + for (const PyMemberDef *memb = it.current.sl_ptr; + memb->name != NULL; + memb++) + { nmembers++; if (memb->flags & Py_RELATIVE_OFFSET) { - if (spec->basicsize > 0) { - PyErr_SetString( - PyExc_SystemError, - "With Py_RELATIVE_OFFSET, basicsize must be negative."); - goto finally; - } - if (memb->offset < 0 || memb->offset >= -spec->basicsize) { + if (memb->offset < 0) { PyErr_SetString( PyExc_SystemError, - "Member offset out of range (0..-basicsize)"); + "Member offset must not be negative"); goto finally; } + have_relative_members = true; + max_relative_offset = Py_MAX(max_relative_offset, + memb->offset); } if (strcmp(memb->name, "__weaklistoffset__") == 0) { weaklistoffset_member = memb; @@ -5345,43 +5391,86 @@ PyType_FromMetaclass( } } break; + case Py_tp_token: + token = it.current.sl_ptr; + if (token == Py_TP_USE_SPEC) { + if (!spec) { + PyErr_SetString( + PyExc_SystemError, + "Py_tp_token: Py_TP_USE_SPEC (NULL) can only be " + "used with PyType_Spec"); + goto finally; + } + token = spec; + } + break; case Py_tp_doc: /* For the docstring slot, which usually points to a static string literal, we need to make a copy */ - if (tp_doc != NULL) { - PyErr_SetString( - PyExc_SystemError, - "Multiple Py_tp_doc slots are not supported."); - goto finally; - } - if (slot->pfunc == NULL) { + if (it.current.sl_ptr == NULL) { PyMem_Free(tp_doc); tp_doc = NULL; } else { - size_t len = strlen(slot->pfunc)+1; + size_t len = strlen(it.current.sl_ptr)+1; tp_doc = PyMem_Malloc(len); if (tp_doc == NULL) { PyErr_NoMemory(); goto finally; } - memcpy(tp_doc, slot->pfunc, len); + memcpy(tp_doc, it.current.sl_ptr, len); } break; } } + #undef NO_SPEC - /* Prepare the type name and qualname */ + /* Required slots & bad combinations */ - if (spec->name == NULL) { - PyErr_SetString(PyExc_SystemError, - "Type spec does not define the name field."); + if (it.name == NULL) { + if (spec) { + PyErr_SetString(PyExc_SystemError, + "Type spec does not define the name field."); + } + else { + PyErr_SetString(PyExc_SystemError, + "Py_tp_name slot is required."); + } goto finally; } - const char *s = strrchr(spec->name, '.'); + if (_PySlotIterator_SawSlot(&it, Py_tp_basicsize) + && _PySlotIterator_SawSlot(&it, Py_tp_extra_basicsize)) + { + PyErr_Format( + PyExc_SystemError, + "type %s: Py_tp_basicsize and Py_tp_extra_basicsize are " + "mutually exclusive", + it.name); + goto finally; + } + + if (have_relative_members) { + if (!extra_basicsize) { + PyErr_SetString( + PyExc_SystemError, + "With Py_RELATIVE_OFFSET, basicsize must be extended"); + goto finally; + } + if (max_relative_offset >= extra_basicsize) { + PyErr_SetString( + PyExc_SystemError, + "Member offset out of range (0..extra_basicsize)"); + goto finally; + } + } + + /* Prepare the type name and qualname */ + + assert(it.name); + const char *s = strrchr(it.name, '.'); if (s == NULL) { - s = spec->name; + s = it.name; } else { s++; @@ -5392,7 +5481,7 @@ PyType_FromMetaclass( goto finally; } - /* Copy spec->name to a buffer we own. + /* Copy the name to a buffer we own. * * Unfortunately, we can't use tp_name directly (with some * flag saying that it should be deallocated with the type), @@ -5401,32 +5490,53 @@ PyType_FromMetaclass( * So, we use a separate buffer, _ht_tpname, that's always * deallocated with the type (if it's non-NULL). */ - Py_ssize_t name_buf_len = strlen(spec->name) + 1; + Py_ssize_t name_buf_len = strlen(it.name) + 1; _ht_tpname = PyMem_Malloc(name_buf_len); if (_ht_tpname == NULL) { + PyErr_NoMemory(); goto finally; } - memcpy(_ht_tpname, spec->name, name_buf_len); + memcpy(_ht_tpname, it.name, name_buf_len); /* Get a tuple of bases. * bases is a strong reference (unlike bases_in). + * (This is convoluted for backwards compatibility -- preserving priority + * of the various ways to specify bases) */ - bases = get_bases_tuple(bases_in, spec); + if (!bases_in) { + bases_in = bases_slot; + } + if (bases_in) { + if (PyTuple_Check(bases_in)) { + bases = Py_NewRef(bases_in); + } + else { + bases = PyTuple_Pack(1, bases_in); + } + } + else { + bases = PyTuple_Pack(1, &PyBaseObject_Type); + } if (!bases) { goto finally; } - /* If this is an immutable type, check if all bases are also immutable, - * and (for now) fire a deprecation warning if not. + /* If this is an immutable type, check if all bases are also immutable. * (This isn't necessary for static types: those can't have heap bases, * and only heap types can be mutable.) */ - if (spec->flags & Py_TPFLAGS_IMMUTABLETYPE) { - if (check_immutable_bases(spec->name, bases, 0) < 0) { + if (flags & Py_TPFLAGS_IMMUTABLETYPE) { + if (check_immutable_bases(it.name, bases, 0) < 0) { goto finally; } } + /* Calculate best base, and check that all bases are type objects */ + PyTypeObject *base = find_best_base(bases); // borrowed ref + if (base == NULL) { + goto finally; + } + /* Calculate the metaclass */ if (!metaclass) { @@ -5449,31 +5559,22 @@ PyType_FromMetaclass( goto finally; } - /* Calculate best base, and check that all bases are type objects */ - PyTypeObject *base = find_best_base(bases); // borrowed ref - if (base == NULL) { - goto finally; - } // find_best_base() should check Py_TPFLAGS_BASETYPE & raise a proper // exception, here we just check its work assert(_PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)); /* Calculate sizes */ - Py_ssize_t basicsize = spec->basicsize; - Py_ssize_t type_data_offset = spec->basicsize; - if (basicsize == 0) { - /* Inherit */ - basicsize = base->tp_basicsize; - } - else if (basicsize < 0) { + Py_ssize_t type_data_offset = basicsize; + if (extra_basicsize) { /* Extend */ + assert(basicsize == 0); type_data_offset = _align_up(base->tp_basicsize); - basicsize = type_data_offset + _align_up(-spec->basicsize); + basicsize = type_data_offset + _align_up(extra_basicsize); /* Inheriting variable-sized types is limited */ if (base->tp_itemsize - && !((base->tp_flags | spec->flags) & Py_TPFLAGS_ITEMS_AT_END)) + && !((base->tp_flags | flags) & Py_TPFLAGS_ITEMS_AT_END)) { PyErr_SetString( PyExc_SystemError, @@ -5481,8 +5582,10 @@ PyType_FromMetaclass( goto finally; } } - - Py_ssize_t itemsize = spec->itemsize; + if (basicsize == 0) { + /* Inherit */ + basicsize = base->tp_basicsize; + } /* Compute special offsets */ @@ -5514,11 +5617,10 @@ PyType_FromMetaclass( if (res == NULL) { goto finally; } - res_start = (char*)res; type = &res->ht_type; /* The flags must be initialized early, before the GC traverses us */ - type_set_flags(type, spec->flags | Py_TPFLAGS_HEAPTYPE); + type_set_flags(type, flags | Py_TPFLAGS_HEAPTYPE); res->ht_module = Py_XNewRef(module); @@ -5547,15 +5649,20 @@ PyType_FromMetaclass( res->_ht_tpname = _ht_tpname; _ht_tpname = NULL; // Give ownership to the type + res->ht_token = token; + /* Copy the sizes */ type->tp_basicsize = basicsize; type->tp_itemsize = itemsize; - /* Copy all the ordinary slots */ + /* Second pass of slots: copy most of them into the type */ - for (slot = spec->slots; slot->slot; slot++) { - switch (slot->slot) { + _PySlotIterator_Rewind(&it, spec ? (void*)spec->slots : (void*)slots); + while (_PySlotIterator_Next(&it)) { + switch (it.current.sl_id) { + case Py_slot_invalid: + goto finally; case Py_tp_base: case Py_tp_bases: case Py_tp_doc: @@ -5565,7 +5672,7 @@ PyType_FromMetaclass( { /* Move the slots to the heap type itself */ size_t len = Py_TYPE(type)->tp_itemsize * nmembers; - memcpy(_PyHeapType_GET_MEMBERS(res), slot->pfunc, len); + memcpy(_PyHeapType_GET_MEMBERS(res), it.current.sl_ptr, len); type->tp_members = _PyHeapType_GET_MEMBERS(res); PyMemberDef *memb; Py_ssize_t i; @@ -5579,26 +5686,8 @@ PyType_FromMetaclass( } } break; - case Py_tp_token: - { - res->ht_token = slot->pfunc == Py_TP_USE_SPEC ? spec : slot->pfunc; - } - break; default: - { - /* Copy other slots directly */ - PySlot_Offset slotoffsets = pyslot_offsets[slot->slot]; - short slot_offset = slotoffsets.slot_offset; - if (slotoffsets.subslot_offset == -1) { - /* Set a slot in the main PyTypeObject */ - *(void**)((char*)res_start + slot_offset) = slot->pfunc; - } - else { - void *procs = *(void**)((char*)res_start + slot_offset); - short subslot_offset = slotoffsets.subslot_offset; - *(void**)((char*)procs + subslot_offset) = slot->pfunc; - } - } + _PySlot_heaptype_apply_field_slot(res, it.current); break; } } @@ -5664,10 +5753,10 @@ PyType_FromMetaclass( goto finally; } if (r == 0) { - s = strrchr(spec->name, '.'); + s = strrchr(it.name, '.'); if (s != NULL) { PyObject *modname = PyUnicode_FromStringAndSize( - spec->name, (Py_ssize_t)(s - spec->name)); + it.name, (Py_ssize_t)(s - it.name)); if (modname == NULL) { goto finally; } @@ -5680,7 +5769,7 @@ PyType_FromMetaclass( else { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "builtin type %.200s has no __module__ attribute", - spec->name)) + it.name)) goto finally; } } @@ -5691,7 +5780,7 @@ PyType_FromMetaclass( ((PyObject*)type)->ob_flags |= _Py_TYPE_REVEALED_FLAG; #endif - finally: +finally: if (PyErr_Occurred()) { Py_CLEAR(res); } @@ -5702,22 +5791,36 @@ PyType_FromMetaclass( return (PyObject*)res; } +PyObject * +PyType_FromSlots(PySlot *slots) +{ + return type_from_slots_or_spec(slots, NULL, NULL, NULL, NULL); +} + +PyObject * +PyType_FromMetaclass( + PyTypeObject *metaclass, PyObject *module, + PyType_Spec *spec, PyObject *bases) +{ + return type_from_slots_or_spec(NULL, spec, metaclass, module, bases); +} + PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, module, spec, bases); + return type_from_slots_or_spec(NULL, spec, NULL, module, bases); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, NULL, spec, bases); + return type_from_slots_or_spec(NULL, spec, NULL, NULL, bases); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return PyType_FromMetaclass(NULL, NULL, spec, NULL); + return type_from_slots_or_spec(NULL, spec, NULL, NULL, NULL); } PyObject * @@ -5739,32 +5842,10 @@ PyType_GetModuleName(PyTypeObject *type) } void * -PyType_GetSlot(PyTypeObject *type, int slot) +PyType_GetSlot(PyTypeObject *type, int slot_in) { - void *parent_slot; - int slots_len = Py_ARRAY_LENGTH(pyslot_offsets); - - if (slot <= 0 || slot >= slots_len) { - PyErr_BadInternalCall(); - return NULL; - } - int slot_offset = pyslot_offsets[slot].slot_offset; - - if (slot_offset >= (int)sizeof(PyTypeObject)) { - if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { - return NULL; - } - } - - parent_slot = *(void**)((char*)type + slot_offset); - if (parent_slot == NULL) { - return NULL; - } - /* Return slot directly if we have no sub slot. */ - if (pyslot_offsets[slot].subslot_offset == -1) { - return parent_slot; - } - return *(void**)((char*)parent_slot + pyslot_offsets[slot].subslot_offset); + uint16_t slot = _PySlot_resolve_type_slot(slot_in); + return _PySlot_type_getslot(type, slot); } PyObject * @@ -6056,7 +6137,7 @@ PyObject_GetItemData(PyObject *obj) static int find_name_in_mro(PyTypeObject *type, PyObject *name, _PyStackRef *out) { - Py_hash_t hash = _PyObject_HashFast(name); + Py_hash_t hash = _PyObject_HashDictKey(name); if (hash == -1) { PyErr_Clear(); return -1; @@ -6417,9 +6498,25 @@ set_flags_recursive(PyTypeObject *self, unsigned long mask, unsigned long flags) void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags) { + BEGIN_TYPE_LOCK(); + /* Ideally, changing flags and invalidating the old version tag would + happen in one step. But type_modified_unlocked() is re-entrant and + cannot run with the world stopped, so we must invalidate first. + Immutable/static-builtin types are skipped because + set_flags_recursive() does not modify them. */ + if (!PyType_HasFeature(self, Py_TPFLAGS_IMMUTABLETYPE) && + (self->tp_flags & mask) != flags) + { + type_modified_unlocked(self); + } + /* Keep TYPE_LOCK held while waiting for stop-the-world so no thread + can reassign a version tag before the flag update. */ + type_lock_prevent_release(); types_stop_world(); set_flags_recursive(self, mask, flags); types_start_world(); + type_lock_allow_release(); + END_TYPE_LOCK(); } /* This is similar to PyObject_GenericGetAttr(), @@ -6685,12 +6782,11 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); #ifdef Py_GIL_DISABLED - // gh-139103: Enable deferred refcounting for functions assigned - // to type objects. This is important for `dataclass.__init__`, - // which is generated dynamically. - if (value != NULL && - PyFunction_Check(value) && - !_PyObject_HasDeferredRefcount(value)) + // gh-139103: Enable deferred refcounting for functions and descriptors + // assigned to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically, and for descriptor scaling on + // free-threaded builds. + if (value != NULL && (PyFunction_Check(value) || Py_TYPE(value)->tp_descr_get != NULL)) { PyUnstable_Object_EnableDeferredRefcount(value); } @@ -6879,6 +6975,33 @@ type_dealloc(PyObject *self) // Assert this is a heap-allocated type object _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); + // Notify type watchers before teardown. The type object is still fully + // intact at this point (dict, bases, mro, name are all valid), so + // callbacks can safely inspect it. + if (type->tp_watched) { + _PyObject_ResurrectStart(self); + PyInterpreterState *interp = _PyInterpreterState_GET(); + int bits = type->tp_watched; + int i = 0; + while (bits) { + assert(i < TYPE_MAX_WATCHERS); + if (bits & 1) { + PyType_WatchCallback cb = interp->type_watchers[i]; + if (cb && (cb(type) < 0)) { + PyErr_FormatUnraisable( + "Exception ignored in type watcher callback #%d " + "for %R", + i, type); + } + } + i++; + bits >>= 1; + } + if (_PyObject_ResurrectEnd(self)) { + return; // callback resurrected the object + } + } + _PyObject_GC_UNTRACK(type); type_dealloc_common(type); @@ -6903,7 +7026,7 @@ type_dealloc(PyObject *self) Py_XDECREF(et->ht_qualname); Py_XDECREF(et->ht_slots); if (et->ht_cached_keys) { - _PyDictKeys_DecRef(et->ht_cached_keys); + _PyDict_RemoveKeysForClass(et); } Py_XDECREF(et->ht_module); PyMem_Free(et->_ht_tpname); @@ -10991,14 +11114,22 @@ slot_tp_iternext(PyObject *self) return vectorcall_method(&_Py_ID(__next__), stack, 1); } +int +_PyType_HasSlotTpIternext(PyTypeObject *type) +{ + return type->tp_iternext == slot_tp_iternext; +} + static PyObject * slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) { PyTypeObject *tp = Py_TYPE(self); - PyObject *get; - - get = _PyType_LookupRef(tp, &_Py_ID(__get__)); - if (get == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + _PyCStackRef cref; + _PyThreadState_PushCStackRef(tstate, &cref); + _PyType_LookupStackRefAndVersion(tp, &_Py_ID(__get__), &cref.ref); + if (PyStackRef_IsNull(cref.ref)) { + _PyThreadState_PopCStackRef(tstate, &cref); #ifndef Py_GIL_DISABLED /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -11010,9 +11141,10 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) obj = Py_None; if (type == NULL) type = Py_None; + PyObject *get = PyStackRef_AsPyObjectBorrow(cref.ref); PyObject *stack[3] = {self, obj, type}; PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); - Py_DECREF(get); + _PyThreadState_PopCStackRef(tstate, &cref); return res; } @@ -11203,6 +11335,7 @@ slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) wrapper = PyObject_GC_New(PyBufferWrapper, &_PyBufferWrapper_Type); if (wrapper == NULL) { + PyBuffer_Release(buffer); goto fail; } wrapper->mv = ret; @@ -12817,7 +12950,8 @@ PyDoc_STRVAR(super_doc, "super() -> same as super(__class__, <first argument>)\n" "super(type) -> unbound super object\n" "super(type, obj) -> bound super object; requires isinstance(obj, type)\n" -"super(type, type2) -> bound super object; requires issubclass(type2, type)\n" +"super(type, type2) -> bound super object; requires\n" +" issubclass(type2, type)\n" "Typical use to call a cooperative superclass method:\n" "class C(B):\n" " def meth(self, arg):\n" diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc deleted file mode 100644 index 642160fe0bd8bcc..000000000000000 --- a/Objects/typeslots.inc +++ /dev/null @@ -1,84 +0,0 @@ -/* Generated by typeslots.py */ -{offsetof(PyBufferProcs, bf_getbuffer), offsetof(PyTypeObject, tp_as_buffer)}, -{offsetof(PyBufferProcs, bf_releasebuffer), offsetof(PyTypeObject, tp_as_buffer)}, -{offsetof(PyMappingMethods, mp_ass_subscript), offsetof(PyTypeObject, tp_as_mapping)}, -{offsetof(PyMappingMethods, mp_length), offsetof(PyTypeObject, tp_as_mapping)}, -{offsetof(PyMappingMethods, mp_subscript), offsetof(PyTypeObject, tp_as_mapping)}, -{offsetof(PyNumberMethods, nb_absolute), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_add), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_and), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_bool), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_divmod), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_float), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_floor_divide), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_index), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_add), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_and), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_floor_divide), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_lshift), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_multiply), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_or), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_power), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_remainder), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_rshift), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_subtract), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_true_divide), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_xor), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_int), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_invert), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_lshift), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_multiply), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_negative), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_or), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_positive), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_power), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_remainder), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_rshift), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_subtract), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_true_divide), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_xor), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PySequenceMethods, sq_ass_item), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_concat), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_contains), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_inplace_concat), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_inplace_repeat), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_item), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_length), offsetof(PyTypeObject, tp_as_sequence)}, -{offsetof(PySequenceMethods, sq_repeat), offsetof(PyTypeObject, tp_as_sequence)}, -{-1, offsetof(PyTypeObject, tp_alloc)}, -{-1, offsetof(PyTypeObject, tp_base)}, -{-1, offsetof(PyTypeObject, tp_bases)}, -{-1, offsetof(PyTypeObject, tp_call)}, -{-1, offsetof(PyTypeObject, tp_clear)}, -{-1, offsetof(PyTypeObject, tp_dealloc)}, -{-1, offsetof(PyTypeObject, tp_del)}, -{-1, offsetof(PyTypeObject, tp_descr_get)}, -{-1, offsetof(PyTypeObject, tp_descr_set)}, -{-1, offsetof(PyTypeObject, tp_doc)}, -{-1, offsetof(PyTypeObject, tp_getattr)}, -{-1, offsetof(PyTypeObject, tp_getattro)}, -{-1, offsetof(PyTypeObject, tp_hash)}, -{-1, offsetof(PyTypeObject, tp_init)}, -{-1, offsetof(PyTypeObject, tp_is_gc)}, -{-1, offsetof(PyTypeObject, tp_iter)}, -{-1, offsetof(PyTypeObject, tp_iternext)}, -{-1, offsetof(PyTypeObject, tp_methods)}, -{-1, offsetof(PyTypeObject, tp_new)}, -{-1, offsetof(PyTypeObject, tp_repr)}, -{-1, offsetof(PyTypeObject, tp_richcompare)}, -{-1, offsetof(PyTypeObject, tp_setattr)}, -{-1, offsetof(PyTypeObject, tp_setattro)}, -{-1, offsetof(PyTypeObject, tp_str)}, -{-1, offsetof(PyTypeObject, tp_traverse)}, -{-1, offsetof(PyTypeObject, tp_members)}, -{-1, offsetof(PyTypeObject, tp_getset)}, -{-1, offsetof(PyTypeObject, tp_free)}, -{offsetof(PyNumberMethods, nb_matrix_multiply), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyNumberMethods, nb_inplace_matrix_multiply), offsetof(PyTypeObject, tp_as_number)}, -{offsetof(PyAsyncMethods, am_await), offsetof(PyTypeObject, tp_as_async)}, -{offsetof(PyAsyncMethods, am_aiter), offsetof(PyTypeObject, tp_as_async)}, -{offsetof(PyAsyncMethods, am_anext), offsetof(PyTypeObject, tp_as_async)}, -{-1, offsetof(PyTypeObject, tp_finalize)}, -{offsetof(PyAsyncMethods, am_send), offsetof(PyTypeObject, tp_as_async)}, -{-1, offsetof(PyTypeObject, tp_vectorcall)}, -{-1, offsetof(PyHeapTypeObject, ht_token)}, diff --git a/Objects/typeslots.py b/Objects/typeslots.py deleted file mode 100755 index c7f8a33bb1e74ea..000000000000000 --- a/Objects/typeslots.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python -# Usage: typeslots.py < Include/typeslots.h typeslots.inc - -import sys, re - - -def generate_typeslots(out=sys.stdout): - out.write("/* Generated by typeslots.py */\n") - res = {} - for line in sys.stdin: - m = re.match("#define Py_([a-z_]+) ([0-9]+)", line) - if not m: - continue - - member = m.group(1) - if member == "tp_token": - # The heap type structure (ht_*) is an implementation detail; - # the public slot for it has a familiar `tp_` prefix - member = '{-1, offsetof(PyHeapTypeObject, ht_token)}' - elif member.startswith("tp_"): - member = f'{{-1, offsetof(PyTypeObject, {member})}}' - elif member.startswith("am_"): - member = (f'{{offsetof(PyAsyncMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_async)}') - elif member.startswith("nb_"): - member = (f'{{offsetof(PyNumberMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_number)}') - elif member.startswith("mp_"): - member = (f'{{offsetof(PyMappingMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_mapping)}') - elif member.startswith("sq_"): - member = (f'{{offsetof(PySequenceMethods, {member}),'+ - ' offsetof(PyTypeObject, tp_as_sequence)}') - elif member.startswith("bf_"): - member = (f'{{offsetof(PyBufferProcs, {member}),'+ - ' offsetof(PyTypeObject, tp_as_buffer)}') - res[int(m.group(2))] = member - - M = max(res.keys())+1 - for i in range(1,M): - if i in res: - out.write("%s,\n" % res[i]) - else: - out.write("{0, 0},\n") - - -def main(): - if len(sys.argv) == 2: - with open(sys.argv[1], "w") as f: - generate_typeslots(f) - else: - generate_typeslots() - -if __name__ == "__main__": - main() diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 8ad590cc6e60936..b2c3c79c93ff195 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -1340,20 +1340,12 @@ paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, PyErr_SetString(PyExc_ValueError, "Variance cannot be specified with infer_variance."); return NULL; } - if (bound != NULL) { - bound = type_check(bound, "Bound must be a type."); - if (bound == NULL) { - return NULL; - } - } PyObject *module = caller(); if (module == NULL) { - Py_XDECREF(bound); return NULL; } PyObject *ps = (PyObject *)paramspec_alloc( name, bound, default_value, covariant, contravariant, infer_variance, module); - Py_XDECREF(bound); Py_DECREF(module); return ps; } @@ -1634,23 +1626,12 @@ typevartuple_impl(PyTypeObject *type, PyObject *name, PyObject *bound, PyErr_SetString(PyExc_ValueError, "Variance cannot be specified with infer_variance."); return NULL; } - if (Py_IsNone(bound)) { - bound = NULL; - } - if (bound != NULL) { - bound = type_check(bound, "Bound must be a type."); - if (bound == NULL) { - return NULL; - } - } PyObject *module = caller(); if (module == NULL) { - Py_XDECREF(bound); return NULL; } PyObject *result = (PyObject *)typevartuple_alloc( name, bound, default_value, covariant, contravariant, infer_variance, module); - Py_XDECREF(bound); Py_DECREF(module); return result; } @@ -2311,8 +2292,9 @@ PyDoc_STRVAR(generic_class_getitem_doc, "Parameterizes a generic class.\n\ \n\ At least, parameterizing a generic class is the *main* thing this\n\ -method does. For example, for some generic class `Foo`, this is called\n\ -when we do `Foo[int]` - there, with `cls=Foo` and `params=int`.\n\ +method does. For example, for some generic class `Foo`, this is\n\ +called when we do `Foo[int]` - there, with `cls=Foo` and\n\ +`params=int`.\n\ \n\ However, note that this method is also called when defining generic\n\ classes in the first place with `class Foo[T]: ...`.\n\ diff --git a/Objects/unicode_format.c b/Objects/unicode_format.c index e2790c8c1d4343d..1d6f3f7d9a6f6a0 100644 --- a/Objects/unicode_format.c +++ b/Objects/unicode_format.c @@ -159,8 +159,13 @@ formatfloat(PyObject *v, return -1; } } - else + else { *p_output = _PyUnicode_FromASCII(p, len); + if (*p_output == NULL) { + PyMem_Free(p); + return -1; + } + } PyMem_Free(p); return 0; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 9aee7120c811de8..f26f9062793d2f4 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -41,7 +41,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_bytes_methods.h" // _Py_bytes_lower() -#include "pycore_bytesobject.h" // _PyBytes_Repeat() +#include "pycore_bytesobject.h" // _PyBytes_RepeatBuffer() #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_codecs.h" // _PyCodec_Lookup() #include "pycore_critical_section.h" // Py_*_CRITICAL_SECTION_SEQUENCE_FAST @@ -10794,35 +10794,33 @@ replace(PyObject *self, PyObject *str1, /* --- Unicode Object Methods --------------------------------------------- */ /*[clinic input] -@permit_long_docstring_body str.title as unicode_title Return a version of the string where each word is titlecased. -More specifically, words start with uppercased characters and all remaining -cased characters have lower case. +More specifically, words start with uppercased characters and all +remaining cased characters have lower case. [clinic start generated code]*/ static PyObject * unicode_title_impl(PyObject *self) -/*[clinic end generated code: output=c75ae03809574902 input=533ce0eb6a7f5d1b]*/ +/*[clinic end generated code: output=c75ae03809574902 input=2a07e2c7df94627a]*/ { return case_operation(self, do_title); } /*[clinic input] -@permit_long_docstring_body str.capitalize as unicode_capitalize Return a capitalized version of the string. -More specifically, make the first character have upper case and the rest lower -case. +More specifically, make the first character have upper case and the +rest lower case. [clinic start generated code]*/ static PyObject * unicode_capitalize_impl(PyObject *self) -/*[clinic end generated code: output=e49a4c333cdb7667 input=a4a15ade41f6f9e9]*/ +/*[clinic end generated code: output=e49a4c333cdb7667 input=e50e50ed45a654cf]*/ { if (PyUnicode_GET_LENGTH(self) == 0) return unicode_result_unchanged(self); @@ -10876,12 +10874,13 @@ str.center as unicode_center Return a centered string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_center_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=420c8859effc7c0c input=b42b247eb26e6519]*/ +/*[clinic end generated code: output=420c8859effc7c0c input=df91017dfd186a78]*/ { Py_ssize_t marg, left; @@ -11441,13 +11440,14 @@ str.count as unicode_count -> Py_ssize_t Return the number of non-overlapping occurrences of substring sub in string S[start:end]. -Optional arguments start and end are interpreted as in slice notation. +Optional arguments start and end are interpreted as in slice +notation. [clinic start generated code]*/ static Py_ssize_t unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=8fcc3aef0b18edbf input=8590716ee228b935]*/ +/*[clinic end generated code: output=8fcc3aef0b18edbf input=c9209e05438cc352]*/ { assert(PyUnicode_Check(str)); assert(PyUnicode_Check(substr)); @@ -11520,8 +11520,8 @@ str.encode as unicode_encode errors: str(c_default="NULL") = 'strict' The error handling scheme to use for encoding errors. The default is 'strict' meaning that encoding errors raise a - UnicodeEncodeError. Other possible values are 'ignore', 'replace' and - 'xmlcharrefreplace' as well as any other name registered with + UnicodeEncodeError. Other possible values are 'ignore', 'replace' + and 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle UnicodeEncodeErrors. Encode the string using the codec registered for encoding. @@ -11529,7 +11529,7 @@ Encode the string using the codec registered for encoding. static PyObject * unicode_encode_impl(PyObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=bf78b6e2a9470e3c input=f0a9eb293d08fe02]*/ +/*[clinic end generated code: output=bf78b6e2a9470e3c input=b85a9645cb33b729]*/ { return PyUnicode_AsEncodedString(self, encoding, errors); } @@ -11626,14 +11626,14 @@ str.find as unicode_find = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Return -1 on failure. +Optional arguments start and end are interpreted as in slice +notation. Return -1 on failure. [clinic start generated code]*/ static Py_ssize_t unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=51dbe6255712e278 input=3a9d650fe4c24695]*/ +/*[clinic end generated code: output=51dbe6255712e278 input=f57e93c59d1ee927]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result < 0) { @@ -11690,14 +11690,14 @@ str.index as unicode_index = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Raises ValueError when the substring is not found. +Optional arguments start and end are interpreted as in slice +notation. Raises ValueError when the substring is not found. [clinic start generated code]*/ static Py_ssize_t unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=77558288837cdf40 input=ae5e48f69ed75b06]*/ +/*[clinic end generated code: output=77558288837cdf40 input=5900ab84de55e628]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result == -1) { @@ -11710,6 +11710,7 @@ unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, } /*[clinic input] +@permit_long_summary str.isascii as unicode_isascii Return True if all characters in the string are ASCII, False otherwise. @@ -11720,24 +11721,23 @@ Empty string is ASCII too. static PyObject * unicode_isascii_impl(PyObject *self) -/*[clinic end generated code: output=c5910d64b5a8003f input=5a43cbc6399621d5]*/ +/*[clinic end generated code: output=c5910d64b5a8003f input=dc74e1ced821159f]*/ { return PyBool_FromLong(PyUnicode_IS_ASCII(self)); } /*[clinic input] -@permit_long_docstring_body str.islower as unicode_islower Return True if the string is a lowercase string, False otherwise. -A string is lowercase if all cased characters in the string are lowercase and -there is at least one cased character in the string. +A string is lowercase if all cased characters in the string are +lowercase and there is at least one cased character in the string. [clinic start generated code]*/ static PyObject * unicode_islower_impl(PyObject *self) -/*[clinic end generated code: output=dbd41995bd005b81 input=c6fc0295241a1aaa]*/ +/*[clinic end generated code: output=dbd41995bd005b81 input=1879b48dfc628366]*/ { Py_ssize_t i, length; int kind; @@ -11770,18 +11770,17 @@ unicode_islower_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isupper as unicode_isupper Return True if the string is an uppercase string, False otherwise. -A string is uppercase if all cased characters in the string are uppercase and -there is at least one cased character in the string. +A string is uppercase if all cased characters in the string are +uppercase and there is at least one cased character in the string. [clinic start generated code]*/ static PyObject * unicode_isupper_impl(PyObject *self) -/*[clinic end generated code: output=049209c8e7f15f59 input=8d5cb33e67efde72]*/ +/*[clinic end generated code: output=049209c8e7f15f59 input=77d29904aef0e3a0]*/ { Py_ssize_t i, length; int kind; @@ -11870,18 +11869,17 @@ unicode_istitle_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isspace as unicode_isspace Return True if the string is a whitespace string, False otherwise. -A string is whitespace if all characters in the string are whitespace and there -is at least one character in the string. +A string is whitespace if all characters in the string are +whitespace and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isspace_impl(PyObject *self) -/*[clinic end generated code: output=163a63bfa08ac2b9 input=44fe05e248c6e159]*/ +/*[clinic end generated code: output=163a63bfa08ac2b9 input=29e09560fc23fbeb]*/ { Py_ssize_t i, length; int kind; @@ -11909,18 +11907,17 @@ unicode_isspace_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isalpha as unicode_isalpha Return True if the string is an alphabetic string, False otherwise. -A string is alphabetic if all characters in the string are alphabetic and there -is at least one character in the string. +A string is alphabetic if all characters in the string are +alphabetic and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isalpha_impl(PyObject *self) -/*[clinic end generated code: output=cc81b9ac3883ec4f input=c233000624a56e0d]*/ +/*[clinic end generated code: output=cc81b9ac3883ec4f input=9906a07f3e04892e]*/ { Py_ssize_t i, length; int kind; @@ -11947,18 +11944,18 @@ unicode_isalpha_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary str.isalnum as unicode_isalnum Return True if the string is an alpha-numeric string, False otherwise. -A string is alpha-numeric if all characters in the string are alpha-numeric and -there is at least one character in the string. +A string is alpha-numeric if all characters in the string are +alpha-numeric and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isalnum_impl(PyObject *self) -/*[clinic end generated code: output=a5a23490ffc3660c input=5d63ba9c9bafdb6b]*/ +/*[clinic end generated code: output=a5a23490ffc3660c input=892f64ebc171fd4f]*/ { int kind; const void *data; @@ -11987,18 +11984,17 @@ unicode_isalnum_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isdecimal as unicode_isdecimal Return True if the string is a decimal string, False otherwise. -A string is a decimal string if all characters in the string are decimal and -there is at least one character in the string. +A string is a decimal string if all characters in the string are +decimal and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isdecimal_impl(PyObject *self) -/*[clinic end generated code: output=fb2dcdb62d3fc548 input=8e84a58b414935a3]*/ +/*[clinic end generated code: output=fb2dcdb62d3fc548 input=63b0453c48cad0af]*/ { Py_ssize_t i, length; int kind; @@ -12025,18 +12021,17 @@ unicode_isdecimal_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isdigit as unicode_isdigit Return True if the string is a digit string, False otherwise. -A string is a digit string if all characters in the string are digits and there -is at least one character in the string. +A string is a digit string if all characters in the string are +digits and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isdigit_impl(PyObject *self) -/*[clinic end generated code: output=10a6985311da6858 input=99e284affb54d4a0]*/ +/*[clinic end generated code: output=10a6985311da6858 input=353b03747b062e4b]*/ { Py_ssize_t i, length; int kind; @@ -12064,18 +12059,17 @@ unicode_isdigit_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.isnumeric as unicode_isnumeric Return True if the string is a numeric string, False otherwise. -A string is numeric if all characters in the string are numeric and there is at -least one character in the string. +A string is numeric if all characters in the string are numeric and +there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isnumeric_impl(PyObject *self) -/*[clinic end generated code: output=9172a32d9013051a input=e9f5b6b8b29b0ee6]*/ +/*[clinic end generated code: output=9172a32d9013051a input=83b2a072ed7aff48]*/ { Py_ssize_t i, length; int kind; @@ -12145,18 +12139,18 @@ PyUnicode_IsIdentifier(PyObject *self) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary str.isidentifier as unicode_isidentifier Return True if the string is a valid Python identifier, False otherwise. -Call keyword.iskeyword(s) to test whether string s is a reserved identifier, -such as "def" or "class". +Call keyword.iskeyword(s) to test whether string s is a reserved +identifier, such as "def" or "class". [clinic start generated code]*/ static PyObject * unicode_isidentifier_impl(PyObject *self) -/*[clinic end generated code: output=fe585a9666572905 input=86315dd889d7bd04]*/ +/*[clinic end generated code: output=fe585a9666572905 input=cabde62c20a3be6b]*/ { return PyBool_FromLong(PyUnicode_IsIdentifier(self)); } @@ -12196,7 +12190,6 @@ unicode_isprintable_impl(PyObject *self) } /*[clinic input] -@permit_long_docstring_body str.join as unicode_join iterable: object @@ -12204,15 +12197,15 @@ str.join as unicode_join Concatenate any number of strings. -The string whose method is called is inserted in between each given string. -The result is returned as a new string. +The string whose method is called is inserted in between each given +string. The result is returned as a new string. Example: '.'.join(['ab', 'pq', 'rs']) -> 'ab.pq.rs' [clinic start generated code]*/ static PyObject * unicode_join(PyObject *self, PyObject *iterable) -/*[clinic end generated code: output=6857e7cecfe7bf98 input=bac724ed412ef3f8]*/ +/*[clinic end generated code: output=6857e7cecfe7bf98 input=fd330a11ee845fb2]*/ { return PyUnicode_Join(self, iterable); } @@ -12232,12 +12225,13 @@ str.ljust as unicode_ljust Return a left-justified string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_ljust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=3ab599e335e60a32]*/ +/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=8a55f06694c20ed6]*/ { if (PyUnicode_GET_LENGTH(self) >= width) return unicode_result_unchanged(self); @@ -12502,8 +12496,8 @@ unicode_rstrip_impl(PyObject *self, PyObject *chars) } -static PyObject* -unicode_repeat(PyObject *str, Py_ssize_t len) +PyObject * +_PyUnicode_Repeat(PyObject *str, Py_ssize_t len) { PyObject *u; Py_ssize_t nchars, n; @@ -12548,7 +12542,7 @@ unicode_repeat(PyObject *str, Py_ssize_t len) else { Py_ssize_t char_size = PyUnicode_KIND(str); char *to = (char *) PyUnicode_DATA(u); - _PyBytes_Repeat(to, nchars * char_size, PyUnicode_DATA(str), + _PyBytes_RepeatBuffer(to, nchars * char_size, PyUnicode_DATA(str), PyUnicode_GET_LENGTH(str) * char_size); } @@ -12593,7 +12587,6 @@ unicode_replace_impl(PyObject *self, PyObject *old, PyObject *new, } /*[clinic input] -@permit_long_docstring_body str.removeprefix as unicode_removeprefix prefix: unicode @@ -12601,13 +12594,14 @@ str.removeprefix as unicode_removeprefix Return a str with the given prefix string removed if present. -If the string starts with the prefix string, return string[len(prefix):]. -Otherwise, return a copy of the original string. +If the string starts with the prefix string, return +string[len(prefix):]. Otherwise, return a copy of the original +string. [clinic start generated code]*/ static PyObject * unicode_removeprefix_impl(PyObject *self, PyObject *prefix) -/*[clinic end generated code: output=f1e5945e9763bcb9 input=1989a856dbb813f1]*/ +/*[clinic end generated code: output=f1e5945e9763bcb9 input=90d162724944bfa7]*/ { int match = tailmatch(self, prefix, 0, PY_SSIZE_T_MAX, -1); if (match == -1) { @@ -12628,14 +12622,14 @@ str.removesuffix as unicode_removesuffix Return a str with the given suffix string removed if present. -If the string ends with the suffix string and that suffix is not empty, -return string[:-len(suffix)]. Otherwise, return a copy of the original -string. +If the string ends with the suffix string and that suffix is not +empty, return string[:-len(suffix)]. Otherwise, return a copy of +the original string. [clinic start generated code]*/ static PyObject * unicode_removesuffix_impl(PyObject *self, PyObject *suffix) -/*[clinic end generated code: output=d36629e227636822 input=12cc32561e769be4]*/ +/*[clinic end generated code: output=d36629e227636822 input=6efc96152d4bfcd5]*/ { int match = tailmatch(self, suffix, 0, PY_SSIZE_T_MAX, +1); if (match == -1) { @@ -12745,14 +12739,14 @@ str.rfind as unicode_rfind = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Return -1 on failure. +Optional arguments start and end are interpreted as in slice +notation. Return -1 on failure. [clinic start generated code]*/ static Py_ssize_t unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=880b29f01dd014c8 input=7f7e97d5cd3299a2]*/ +/*[clinic end generated code: output=880b29f01dd014c8 input=2e67789533baf2f5]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result < 0) { @@ -12767,14 +12761,14 @@ str.rindex as unicode_rindex = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Raises ValueError when the substring is not found. +Optional arguments start and end are interpreted as in slice +notation. Raises ValueError when the substring is not found. [clinic start generated code]*/ static Py_ssize_t unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=5f3aef124c867fe1 input=0363a324740b3e62]*/ +/*[clinic end generated code: output=5f3aef124c867fe1 input=e29d446c8234c9d9]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result == -1) { @@ -12795,12 +12789,13 @@ str.rjust as unicode_rjust Return a right-justified string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_rjust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=804a1a57fbe8d5cf input=d05f550b5beb1f72]*/ +/*[clinic end generated code: output=804a1a57fbe8d5cf input=1256a8d659589907]*/ { if (PyUnicode_GET_LENGTH(self) >= width) return unicode_result_unchanged(self); @@ -12824,9 +12819,9 @@ str.split as unicode_split sep: object = None The separator used to split the string. - When set to None (the default value), will split on any whitespace - character (including \n \r \t \f and spaces) and will discard - empty strings from the result. + When set to None (the default value), will split on any + whitespace character (including \n \r \t \f and spaces) and + will discard empty strings from the result. maxsplit: Py_ssize_t = -1 Maximum number of splits. -1 (the default value) means no limit. @@ -12835,15 +12830,15 @@ Return a list of the substrings in the string, using sep as the separator string Splitting starts at the front of the string and works to the end. -Note, str.split() is mainly useful for data that has been intentionally -delimited. With natural text that includes punctuation, consider using -the regular expression module. +Note, str.split() is mainly useful for data that has been +intentionally delimited. With natural text that includes +punctuation, consider using the regular expression module. [clinic start generated code]*/ static PyObject * unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=2c1fd08a78e038b8]*/ +/*[clinic end generated code: output=3a65b1db356948dc input=288cfd6bc8828f5a]*/ { if (sep == Py_None) return split(self, NULL, maxsplit); @@ -12960,7 +12955,6 @@ PyUnicode_RPartition(PyObject *str_obj, PyObject *sep_obj) } /*[clinic input] -@permit_long_docstring_body str.partition as unicode_partition sep: object @@ -12968,38 +12962,38 @@ str.partition as unicode_partition Partition the string into three parts using the given separator. -This will search for the separator in the string. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. +This will search for the separator in the string. If the separator +is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it. -If the separator is not found, returns a 3-tuple containing the original string -and two empty strings. +If the separator is not found, returns a 3-tuple containing +the original string and two empty strings. [clinic start generated code]*/ static PyObject * unicode_partition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=e4ced7bd253ca3c4 input=4d854b520d7b0e97]*/ +/*[clinic end generated code: output=e4ced7bd253ca3c4 input=e45faa8c26270cb1]*/ { return PyUnicode_Partition(self, sep); } /*[clinic input] -@permit_long_docstring_body str.rpartition as unicode_rpartition = str.partition Partition the string into three parts using the given separator. -This will search for the separator in the string, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. +This will search for the separator in the string, starting at the +end. If the separator is found, returns a 3-tuple containing the +part before the separator, the separator itself, and the part after +it. -If the separator is not found, returns a 3-tuple containing two empty strings -and the original string. +If the separator is not found, returns a 3-tuple containing two +empty strings and the original string. [clinic start generated code]*/ static PyObject * unicode_rpartition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=1aa13cf1156572aa input=a6adabe91e75b486]*/ +/*[clinic end generated code: output=1aa13cf1156572aa input=53a7f8cb19975b7c]*/ { return PyUnicode_RPartition(self, sep); } @@ -13038,20 +13032,20 @@ unicode_rsplit_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary str.splitlines as unicode_splitlines keepends: bool = False Return a list of the lines in the string, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * unicode_splitlines_impl(PyObject *self, int keepends) -/*[clinic end generated code: output=f664dcdad153ec40 input=39eeafbfef61c827]*/ +/*[clinic end generated code: output=f664dcdad153ec40 input=b45ea0f87645a06d]*/ { return PyUnicode_Splitlines(self, keepends); } @@ -13130,18 +13124,19 @@ str.maketrans as unicode_maketrans Return a translation table usable for str.translate(). -If there is only one argument, it must be a dictionary mapping Unicode -ordinals (integers) or characters to Unicode ordinals, strings or None. -Character keys will be then converted to ordinals. -If there are two arguments, they must be strings of equal length, and -in the resulting dictionary, each character in x will be mapped to the -character at the same position in y. If there is a third argument, it -must be a string, whose characters will be mapped to None in the result. +If there is only one argument, it must be a dictionary mapping +Unicode ordinals (integers) or characters to Unicode ordinals, +strings or None. Character keys will be then converted to ordinals. +If there are two arguments, they must be strings of equal length, +and in the resulting dictionary, each character in x will be mapped +to the character at the same position in y. If there is a third +argument, it must be a string, whose characters will be mapped to +None in the result. [clinic start generated code]*/ static PyObject * unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) -/*[clinic end generated code: output=a925c89452bd5881 input=7bfbf529a293c6c5]*/ +/*[clinic end generated code: output=a925c89452bd5881 input=66bc00a1b4258a6e]*/ { PyObject *new = NULL, *key, *value; Py_ssize_t i = 0; @@ -13221,24 +13216,25 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary str.translate as unicode_translate table: object - Translation table, which must be a mapping of Unicode ordinals to - Unicode ordinals, strings, or None. + Translation table, which must be a mapping of Unicode ordinals + to Unicode ordinals, strings, or None. / Replace each character in the string using the given translation table. -The table must implement lookup/indexing via __getitem__, for instance a -dictionary or list. If this operation raises LookupError, the character is -left untouched. Characters mapped to None are deleted. +The table must implement lookup/indexing via __getitem__, for +instance a dictionary or list. If this operation raises +LookupError, the character is left untouched. Characters mapped to +None are deleted. [clinic start generated code]*/ static PyObject * unicode_translate(PyObject *self, PyObject *table) -/*[clinic end generated code: output=3cb448ff2fd96bf3 input=699e5fa0ebf9f5e9]*/ +/*[clinic end generated code: output=3cb448ff2fd96bf3 input=48cf0efe06bc1b75]*/ { return _PyUnicode_TranslateCharmap(self, table, "ignore"); } @@ -13434,6 +13430,7 @@ Return a formatted version of the string, using substitutions from mapping.\n\ The substitutions are identified by braces ('{' and '}')."); /*[clinic input] +@permit_long_summary str.__format__ as unicode___format__ format_spec: unicode @@ -13444,7 +13441,7 @@ Return a formatted version of the string as described by format_spec. static PyObject * unicode___format___impl(PyObject *self, PyObject *format_spec) -/*[clinic end generated code: output=45fceaca6d2ba4c8 input=5e135645d167a214]*/ +/*[clinic end generated code: output=45fceaca6d2ba4c8 input=77a2a19f3f7969f2]*/ { _PyUnicodeWriter writer; int ret; @@ -13510,8 +13507,8 @@ of all lines in the [src, end). It returns the length of the common leading whitespace and sets `output` to point to the beginning of the common leading whitespace if length > 0. */ -static Py_ssize_t -search_longest_common_leading_whitespace( +Py_ssize_t +_Py_search_longest_common_leading_whitespace( const char *const src, const char *const end, const char **output) @@ -13607,7 +13604,7 @@ _PyUnicode_Dedent(PyObject *unicode) // [whitespace_start, whitespace_start + whitespace_len) // describes the current longest common leading whitespace const char *whitespace_start = NULL; - Py_ssize_t whitespace_len = search_longest_common_leading_whitespace( + Py_ssize_t whitespace_len = _Py_search_longest_common_leading_whitespace( src, end, &whitespace_start); if (whitespace_len == 0) { @@ -13734,7 +13731,7 @@ static PyNumberMethods unicode_as_number = { static PySequenceMethods unicode_as_sequence = { unicode_length, /* sq_length */ PyUnicode_Concat, /* sq_concat */ - unicode_repeat, /* sq_repeat */ + _PyUnicode_Repeat, /* sq_repeat */ unicode_getitem, /* sq_item */ 0, /* sq_slice */ 0, /* sq_ass_item */ @@ -14928,33 +14925,6 @@ _PyUnicode_FiniEncodings(struct _Py_unicode_fs_codec *fs_codec) } -#ifdef MS_WINDOWS -int -_PyUnicode_EnableLegacyWindowsFSEncoding(void) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyConfig *config = (PyConfig *)_PyInterpreterState_GetConfig(interp); - - /* Set the filesystem encoding to mbcs/replace (PEP 529) */ - wchar_t *encoding = _PyMem_RawWcsdup(L"mbcs"); - wchar_t *errors = _PyMem_RawWcsdup(L"replace"); - if (encoding == NULL || errors == NULL) { - PyMem_RawFree(encoding); - PyMem_RawFree(errors); - PyErr_NoMemory(); - return -1; - } - - PyMem_RawFree(config->filesystem_encoding); - config->filesystem_encoding = encoding; - PyMem_RawFree(config->filesystem_errors); - config->filesystem_errors = errors; - - return init_fs_codec(interp); -} -#endif - - #ifdef Py_DEBUG static inline int unicode_is_finalizing(void) diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 0f6b1e44bc2402c..1dc2927b6e6ac70 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -518,7 +518,8 @@ union_mro_entries(PyObject *self, PyObject *args) static PyMethodDef union_methods[] = { {"__mro_entries__", union_mro_entries, METH_O}, - {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, + PyDoc_STR("Create a union containing the given types")}, {0} }; diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 61fa3ddad0bfd83..daeba5368b21660 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -416,8 +416,15 @@ get_or_create_weakref(PyTypeObject *type, PyObject *obj, PyObject *callback) Py_TYPE(obj)->tp_name); return NULL; } - if (callback == Py_None) + if (callback == Py_None) { callback = NULL; + } + if (callback != NULL && !PyCallable_Check(callback)) { + PyErr_Format(PyExc_TypeError, + "callback must be callable or None, not '%T'", + callback); + return NULL; + } PyWeakReference **list = GET_WEAKREFS_LISTPTR(obj); if ((type == &_PyWeakref_RefType) || @@ -491,7 +498,8 @@ static PyMemberDef weakref_members[] = { static PyMethodDef weakref_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Weakrefs are generic over the type of the referenced object.")}, {NULL} /* Sentinel */ }; diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index 86df2c7183c30d5..b9a229b1398ec8d 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -224,20 +224,19 @@ wait_event(HANDLE event, DWORD timeout) /*[clinic input] -@permit_long_docstring_body _wmi.exec_query query: unicode Runs a WMI query against the local machine. -This returns a single string with 'name=value' pairs in a flat array separated -by null characters. +This returns a single string with 'name=value' pairs in a flat array +separated by null characters. [clinic start generated code]*/ static PyObject * _wmi_exec_query_impl(PyObject *module, PyObject *query) -/*[clinic end generated code: output=a62303d5bb5e003f input=621f5c50c56d06d0]*/ +/*[clinic end generated code: output=a62303d5bb5e003f input=a8d5710acdfbf515]*/ /*[clinic end generated code]*/ { diff --git a/PC/clinic/_wmimodule.cpp.h b/PC/clinic/_wmimodule.cpp.h index 38d52d0329dcc0d..6c18990f056b5f7 100644 --- a/PC/clinic/_wmimodule.cpp.h +++ b/PC/clinic/_wmimodule.cpp.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_wmi_exec_query__doc__, "\n" "Runs a WMI query against the local machine.\n" "\n" -"This returns a single string with \'name=value\' pairs in a flat array separated\n" -"by null characters."); +"This returns a single string with \'name=value\' pairs in a flat array\n" +"separated by null characters."); #define _WMI_EXEC_QUERY_METHODDEF \ {"exec_query", _PyCFunction_CAST(_wmi_exec_query), METH_FASTCALL|METH_KEYWORDS, _wmi_exec_query__doc__}, @@ -72,4 +72,4 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj exit: return return_value; } -/*[clinic end generated code: output=802bcbcba69e8d0e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f246d0e568cc2d2c input=a9049054013a1b77]*/ diff --git a/PC/launcher.c b/PC/launcher.c index fed5e156b92cb3e..5667fc851aa5904 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1273,6 +1273,7 @@ static PYC_MAGIC magic_values[] = { { 3550, 3599, L"3.13" }, { 3600, 3649, L"3.14" }, { 3650, 3699, L"3.15" }, + { 3700, 3749, L"3.16" }, { 0 } }; diff --git a/PC/layout/main.py b/PC/layout/main.py index 8543e7c56e1c419..f70a26b2b296591 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -22,6 +22,7 @@ __path__ = [str(Path(__file__).resolve().parent)] from .support.appxmanifest import * +from .support.builddetails import * from .support.catalog import * from .support.constants import * from .support.filesets import * @@ -32,7 +33,8 @@ from .support.pymanager import * from .support.nuspec import * -TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_35", "_ctypes_test", "_test*") +TEST_PYDS_ONLY = FileStemSet("xxlimited", "xxlimited_3_13", "xxlimited_35", + "_ctypes_test", "_test*") TEST_DLLS_ONLY = set() TEST_DIRS_ONLY = FileNameSet("test", "tests") @@ -127,7 +129,10 @@ def get_tcltk_lib(ns): def get_layout(ns): def in_build(f, dest="", new_name=None, no_lib=False): n, _, x = f.rpartition(".") - n = new_name or n + if new_name and new_name.endswith(f".{x}"): + n = new_name.rpartition(".")[0] + else: + n = new_name or n src = ns.build / f if ns.debug and src not in REQUIRED_DLLS: if not "_d." in src.name: @@ -161,11 +166,12 @@ def in_build(f, dest="", new_name=None, no_lib=False): source = "python_uwp.exe" sourcew = "pythonw_uwp.exe" elif ns.include_freethreaded: - source = "python{}t.exe".format(VER_DOT) - sourcew = "pythonw{}t.exe".format(VER_DOT) if not ns.include_alias: alias = [] aliasw = [] + if (VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4) < (3, 15, 0, 0xB0): + source = "python{}t.exe".format(VER_DOT) + sourcew = "pythonw{}t.exe".format(VER_DOT) alias.extend([ "python{}t".format(VER_DOT), "python{}t".format(VER_MAJOR) if ns.include_alias3 else None, @@ -196,6 +202,8 @@ def in_build(f, dest="", new_name=None, no_lib=False): yield from in_build(FREETHREADED_PYTHON_STABLE_DLL_NAME) else: yield from in_build(PYTHON_STABLE_DLL_NAME) + if (VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4) >= (3, 15, 0, 0xB0): + yield from in_build(FREETHREADED_PYTHON_STABLE_DLL_NAME) found_any = False for dest, src in rglob(ns.build, "vcruntime*.dll"): @@ -310,6 +318,9 @@ def _c(d): for dest, src in get_appx_layout(ns): yield dest, src + for dest, src in get_builddetails(ns): + yield dest, src + if ns.include_cat: if ns.flat_dlls: yield ns.include_cat.name, ns.include_cat diff --git a/PC/layout/support/builddetails.py b/PC/layout/support/builddetails.py new file mode 100644 index 000000000000000..6ef860eeb043545 --- /dev/null +++ b/PC/layout/support/builddetails.py @@ -0,0 +1,119 @@ +import io +import json +from . import constants + +_LEVELS = { + 0xA0: "alpha", + 0xB0: "beta", + 0xC0: "candidate", + 0xF0: "final", +} + + +_TEMPLATE = { + "schema_version": "1.0", + "base_prefix": ".", + "base_interpreter": "python.exe", + "platform": None, # Set later + "language": { + "version": f"{constants.VER_MAJOR}.{constants.VER_MINOR}", + "version_info": { + "major": constants.VER_MAJOR, + "minor": constants.VER_MINOR, + "micro": constants.VER_MICRO, + "releaselevel": _LEVELS.get(constants.VER_FIELD4 & 0xF0, "final"), + "serial": constants.VER_FIELD4 & 0x0F, + }, + }, + "implementation": { + "name": "cpython", + "cache_tag": f"cpython-{constants.VER_MAJOR}{constants.VER_MINOR}", + "version": { + "major": constants.VER_MAJOR, + "minor": constants.VER_MINOR, + "micro": constants.VER_MICRO, + "releaselevel": _LEVELS.get(constants.VER_FIELD4 & 0xF0, "final"), + "serial": constants.VER_FIELD4 & 0x0F, + }, + "hexversion": constants.VER_HEXVERSION, + }, + "abi": { + "flags": [], + "extension_suffix": ".pyd", + "stable_abi_suffix": ".pyd", + }, + "suffixes": { + "source": [".py", ".pyw"], + "bytecode": [".pyc"], + "extensions": [".pyd"], + }, + "libpython": { + "dynamic": constants.PYTHON_DLL_NAME, + "dynamic_stableabi": constants.PYTHON_STABLE_DLL_NAME, + "link_extensions": True, + }, + "c_api": { + }, +} + + +def _with_d(path): + pre, sep, post = path.partition(".") + return pre + "_d" + sep + post + + +def _add_d(data, *args): + for a in args[:-1]: + data = data[a] + a = args[-1] + v = data[a] + if isinstance(v, list): + data[a] = [_with_d(i) for i in data[a]] + else: + data[a] = _with_d(data[a]) + + +def get_builddetails(ns): + if not ns.include_builddetails_json: + return + + details = dict(_TEMPLATE) + + plat = { + "win32": "win32", + "amd64": "win-amd64", + "arm64": "win-arm64", + }.get(ns.arch, ns.arch) + + pyd_abi_flags = "" + if ns.include_freethreaded: + details["abi"]["flags"].append("t") + pyd_abi_flags += "t" + if ns.debug: + details["abi"]["flags"].append("d") + + norm_plat = plat.replace("-", "_") + ext_suffix = f".cp{constants.VER_MAJOR}{constants.VER_MINOR}{pyd_abi_flags}-{norm_plat}.pyd" + details["abi"]["extension_suffix"] = ext_suffix + details["suffixes"]["extensions"].insert(0, ext_suffix) + + details["platform"] = plat + + if ns.include_dev: + details["c_api"]["headers"] = "Include" + + if ns.include_freethreaded: + details["libpython"]["dynamic"] = constants.FREETHREADED_PYTHON_DLL_NAME + details["libpython"]["dynamic_stableabi"] = constants.FREETHREADED_PYTHON_STABLE_DLL_NAME + + if ns.debug: + _add_d(details, "base_interpreter") + _add_d(details, "abi", "stable_abi_suffix") + _add_d(details, "abi", "extension_suffix") + _add_d(details, "suffixes", "extensions") + _add_d(details, "libpython", "dynamic") + _add_d(details, "libpython", "dynamic_stableabi") + + buffer = io.StringIO() + json.dump(details, buffer, indent=2) + yield "build-details.json", ("build-details.json", buffer.getvalue().encode()) diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index 6b8c915e519743f..cb16f534685c8f8 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -23,6 +23,14 @@ def _unpack_hexversion(): return _read_patchlevel_version(pathlib.Path(os.getenv("PYTHONINCLUDE"))) except OSError: pass + # Manual search for a '-s <source dir>` arument + try: + src = sys.argv[sys.argv.index("-s") + 1] + return _read_patchlevel_version(pathlib.Path(src) / "Include") + except (IndexError, ValueError): + pass + except OSError: + pass return struct.pack(">i", sys.hexversion) @@ -68,6 +76,7 @@ def check_patchlevel_version(sources): VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = _unpack_hexversion() +VER_HEXVERSION = (VER_MAJOR << 24) | (VER_MINOR << 16) | (VER_MICRO << 8) | (VER_FIELD4) VER_SUFFIX = _get_suffix(VER_FIELD4) VER_FIELD3 = VER_MICRO << 8 | VER_FIELD4 VER_DOT = "{}.{}".format(VER_MAJOR, VER_MINOR) diff --git a/PC/layout/support/options.py b/PC/layout/support/options.py index e8c393385425e72..f67d8ba04d90703 100644 --- a/PC/layout/support/options.py +++ b/PC/layout/support/options.py @@ -39,6 +39,7 @@ def public(f): "install-json": {"help": "a PyManager __install__.json file"}, "install-embed-json": {"help": "a PyManager __install__.json file for embeddable distro"}, "install-test-json": {"help": "a PyManager __install__.json for the test distro"}, + "builddetails-json": {"help": "a PEP 739 build-details.json"}, } @@ -69,6 +70,7 @@ def public(f): "props", "nuspec", "alias", + "builddetails-json", ], }, "iot": {"help": "Windows IoT Core", "options": ["alias", "stable", "pip"]}, @@ -85,6 +87,7 @@ def public(f): "symbols", "html-doc", "alias", + "builddetails-json", ], }, "embed": { @@ -96,6 +99,7 @@ def public(f): "flat-dlls", "underpth", "precompile", + "builddetails-json", ], }, "pymanager": { @@ -108,7 +112,9 @@ def public(f): "venv", "dev", "html-doc", + "alias", "install-json", + "builddetails-json", ], }, "pymanager-test": { @@ -123,7 +129,9 @@ def public(f): "html-doc", "symbols", "tests", + "alias", "install-test-json", + "builddetails-json", ], }, } diff --git a/PC/layout/support/pymanager.py b/PC/layout/support/pymanager.py index 831d49ea3f9b46f..f6316e0295c74af 100644 --- a/PC/layout/support/pymanager.py +++ b/PC/layout/support/pymanager.py @@ -66,8 +66,9 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): if ns.include_freethreaded: # Free-threaded distro comes with a tag suffix TAG_SUFFIX = "t" - TARGET = f"python{VER_MAJOR}.{VER_MINOR}t.exe" - TARGETW = f"pythonw{VER_MAJOR}.{VER_MINOR}t.exe" + if not ns.include_alias: + TARGET = f"python{VER_MAJOR}.{VER_MINOR}t.exe" + TARGETW = f"pythonw{VER_MAJOR}.{VER_MINOR}t.exe" DISPLAY_TAGS.append("free-threaded") FILE_SUFFIX = f"t-{ns.arch}" diff --git a/PC/pyconfig.h b/PC/pyconfig.h index a126fca6f5aafb1..2381ed3772b109e 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -330,19 +330,21 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ the linking is explicitly handled */ # if defined(Py_GIL_DISABLED) # if defined(Py_DEBUG) -# pragma comment(lib,"python315t_d.lib") -# elif defined(Py_LIMITED_API) +# pragma comment(lib,"python316t_d.lib") +# elif defined(Py_LIMITED_API) || defined(Py_TARGET_ABI3T) # pragma comment(lib,"python3t.lib") # else -# pragma comment(lib,"python315t.lib") +# pragma comment(lib,"python316t.lib") # endif /* Py_DEBUG */ # else /* Py_GIL_DISABLED */ # if defined(Py_DEBUG) -# pragma comment(lib,"python315_d.lib") +# pragma comment(lib,"python316_d.lib") +# elif defined(Py_TARGET_ABI3T) +# pragma comment(lib,"python3t.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python315.lib") +# pragma comment(lib,"python316.lib") # endif /* Py_DEBUG */ # endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER && !Py_NO_LINK_LIB */ diff --git a/PC/python3dll.c b/PC/python3dll.c index 6b9ef0a4164c168..e0be9d65a93cda6 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -174,6 +174,10 @@ EXPORT_FUNC(PyCodec_XMLCharRefReplaceErrors) EXPORT_FUNC(PyComplex_FromDoubles) EXPORT_FUNC(PyComplex_ImagAsDouble) EXPORT_FUNC(PyComplex_RealAsDouble) +EXPORT_FUNC(PyCriticalSection2_Begin) +EXPORT_FUNC(PyCriticalSection2_End) +EXPORT_FUNC(PyCriticalSection_Begin) +EXPORT_FUNC(PyCriticalSection_End) EXPORT_FUNC(PyDescr_NewClassMethod) EXPORT_FUNC(PyDescr_NewGetSet) EXPORT_FUNC(PyDescr_NewMember) @@ -326,12 +330,18 @@ EXPORT_FUNC(PyImport_ImportModuleLevelObject) EXPORT_FUNC(PyImport_ImportModuleNoBlock) EXPORT_FUNC(PyImport_ReloadModule) EXPORT_FUNC(PyIndex_Check) +EXPORT_FUNC(PyInterpreterGuard_Close) +EXPORT_FUNC(PyInterpreterGuard_FromCurrent) +EXPORT_FUNC(PyInterpreterGuard_FromView) EXPORT_FUNC(PyInterpreterState_Clear) EXPORT_FUNC(PyInterpreterState_Delete) EXPORT_FUNC(PyInterpreterState_Get) EXPORT_FUNC(PyInterpreterState_GetDict) EXPORT_FUNC(PyInterpreterState_GetID) EXPORT_FUNC(PyInterpreterState_New) +EXPORT_FUNC(PyInterpreterView_Close) +EXPORT_FUNC(PyInterpreterView_FromCurrent) +EXPORT_FUNC(PyInterpreterView_FromMain) EXPORT_FUNC(PyIter_Check) EXPORT_FUNC(PyIter_Next) EXPORT_FUNC(PyIter_NextItem) @@ -657,12 +667,15 @@ EXPORT_FUNC(PyThread_tss_set) EXPORT_FUNC(PyThreadState_Clear) EXPORT_FUNC(PyThreadState_Delete) EXPORT_FUNC(PyThreadState_DeleteCurrent) +EXPORT_FUNC(PyThreadState_Ensure) +EXPORT_FUNC(PyThreadState_EnsureFromView) EXPORT_FUNC(PyThreadState_Get) EXPORT_FUNC(PyThreadState_GetDict) EXPORT_FUNC(PyThreadState_GetFrame) EXPORT_FUNC(PyThreadState_GetID) EXPORT_FUNC(PyThreadState_GetInterpreter) EXPORT_FUNC(PyThreadState_New) +EXPORT_FUNC(PyThreadState_Release) EXPORT_FUNC(PyThreadState_SetAsyncExc) EXPORT_FUNC(PyThreadState_Swap) EXPORT_FUNC(PyTraceBack_Here) @@ -677,6 +690,7 @@ EXPORT_FUNC(PyType_ClearCache) EXPORT_FUNC(PyType_Freeze) EXPORT_FUNC(PyType_FromMetaclass) EXPORT_FUNC(PyType_FromModuleAndSpec) +EXPORT_FUNC(PyType_FromSlots) EXPORT_FUNC(PyType_FromSpec) EXPORT_FUNC(PyType_FromSpecWithBases) EXPORT_FUNC(PyType_GenericAlloc) diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index 8cdb8d722cdb9a0..1b44216dc20d1e6 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -13,6 +13,7 @@ #if defined(__clang__) #define _SILENCE_CLANG_COROUTINE_MESSAGE #endif +#define _SILENCE_EXPERIMENTAL_COROUTINE_DEPRECATION_WARNINGS #include <appmodel.h> #include <winrt\Windows.ApplicationModel.h> diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 953973a2ad32df7..17b98c9d9ec3455 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -237,6 +237,7 @@ <ClCompile Include="..\Python\intrinsics.c" /> <ClCompile Include="..\Python\instrumentation.c" /> <ClCompile Include="..\Python\jit.c" /> + <ClCompile Include="..\Python\jit_publish.c" /> <ClCompile Include="..\Python\legacy_tracing.c" /> <ClCompile Include="..\Python\lock.c" /> <ClCompile Include="..\Python\marshal.c" /> @@ -269,6 +270,8 @@ <ClCompile Include="..\Python\pytime.c" /> <ClCompile Include="..\Python\qsbr.c" /> <ClCompile Include="..\Python\remote_debugging.c" /> + <ClCompile Include="..\Python\slots.c" /> + <ClCompile Include="..\Python\slots_generated.c" /> <ClCompile Include="..\Python\specialize.c" /> <ClCompile Include="..\Python\structmember.c" /> <ClCompile Include="..\Python\suggestions.c" /> diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 13db4d93f54518d..af3fded0dabf2d6 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -265,6 +265,9 @@ <ClCompile Include="..\Python\jit.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\jit_publish.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Objects\lazyimportobject.c"> <Filter>Source Files</Filter> </ClCompile> @@ -415,6 +418,12 @@ <ClCompile Include="..\Python\remote_debugging.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Python\slots.c"> + <Filter>Source Files</Filter> + </ClCompile> + <ClCompile Include="..\Python\slots_generated.c"> + <Filter>Source Files</Filter> + </ClCompile> <ClCompile Include="..\Python\specialize.c"> <Filter>Source Files</Filter> </ClCompile> diff --git a/PCbuild/_remote_debugging.vcxproj b/PCbuild/_remote_debugging.vcxproj index 3a9b4033a697ad3..41aadbd13d0995d 100644 --- a/PCbuild/_remote_debugging.vcxproj +++ b/PCbuild/_remote_debugging.vcxproj @@ -128,6 +128,10 @@ <Project>{885d4898-d08d-4091-9c40-c700cfe3fc5a}</Project> <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> + <ProjectReference Include="python3tdll.vcxproj"> + <Project>{947BB5F5-6025-4A4F-8182-1B175469F8D2}</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 68707a54ff6b874..64e50b67be46561 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -133,6 +133,7 @@ <ClCompile Include="..\Modules\_testcapi\frame.c" /> <ClCompile Include="..\Modules\_testcapi\type.c" /> <ClCompile Include="..\Modules\_testcapi\function.c" /> + <ClCompile Include="..\Modules\_testcapi\weakref.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> @@ -146,6 +147,10 @@ <Project>{885d4898-d08d-4091-9c40-c700cfe3fc5a}</Project> <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> + <ProjectReference Include="python3tdll.vcxproj"> + <Project>{947BB5F5-6025-4A4F-8182-1B175469F8D2}</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index b0e75ce433ab14d..a3b62e1df663e00 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -132,6 +132,9 @@ <ClCompile Include="..\Modules\_testcapi\function.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\_testcapi\weakref.c"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc"> diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 935467dfcb32834..69558d204dbb8e7 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -109,6 +109,7 @@ <ClCompile Include="..\Modules\_testlimitedcapi\object.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\pyos.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\set.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\slots.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\sys.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\threadstate.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\tuple.c" /> @@ -116,6 +117,7 @@ <ClCompile Include="..\Modules\_testlimitedcapi\vectorcall_limited.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\version.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\file.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> @@ -129,6 +131,10 @@ <Project>{885d4898-d08d-4091-9c40-c700cfe3fc5a}</Project> <ReferenceOutputAssembly>false</ReferenceOutputAssembly> </ProjectReference> + <ProjectReference Include="python3tdll.vcxproj"> + <Project>{947BB5F5-6025-4A4F-8182-1B175469F8D2}</Project> + <ReferenceOutputAssembly>false</ReferenceOutputAssembly> + </ProjectReference> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 5e0a0f65cfcc3d3..2bcc3f6ff176bd9 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -24,6 +24,7 @@ <ClCompile Include="..\Modules\_testlimitedcapi\object.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\pyos.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\set.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\slots.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\sys.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\testcapi_long.h" /> <ClCompile Include="..\Modules\_testlimitedcapi\threadstate.c" /> @@ -32,6 +33,7 @@ <ClCompile Include="..\Modules\_testlimitedcapi\vectorcall_limited.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\version.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\file.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" /> <ClCompile Include="..\Modules\_testlimitedcapi.c" /> </ItemGroup> <ItemGroup> diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 405285b65dd270a..47399fe65d1e542 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,11 +54,11 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.6 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.7 set libraries=%libraries% mpdecimal-4.0.0 -set libraries=%libraries% sqlite-3.50.4.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 +set libraries=%libraries% sqlite-3.53.2.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.3.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.3.1 set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 @@ -79,8 +79,8 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.6 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.7 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.3.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 7a5327bf016cea9..9d077bbd3f0ba27 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -61,6 +61,8 @@ </Projects> <!-- python3.dll --> <Projects Include="python3dll.vcxproj" /> + <!-- python3t.dll --> + <Projects Include="python3tdll.vcxproj" /> <!-- py[w].exe --> <Projects Include="pylauncher.vcxproj;pywlauncher.vcxproj" /> <!-- pyshellext.dll --> @@ -82,6 +84,7 @@ <TestModules Include="_ctypes_test;_testbuffer;_testcapi;_testlimitedcapi;_testinternalcapi;_testembed;_testimportmultiple;_testmultiphase;_testsinglephase;_testconsole;_testclinic;_testclinic_limited" /> <TestModules Include="xxlimited" Condition="'$(Configuration)' == 'Release'" /> <TestModules Include="xxlimited_35" Condition="'$(Configuration)' == 'Release'" /> + <TestModules Include="xxlimited_3_13" Condition="'$(Configuration)' == 'Release'" /> <Projects Include="@(TestModules->'%(Identity).vcxproj')" Condition="$(IncludeTests)"> <!-- Disable parallel build for test modules --> <BuildInParallel>false</BuildInParallel> diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 7296ea75301157d..09a989d38648df4 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -33,6 +33,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python", "python.vcxproj", {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} = {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} {86937F53-C189-40EF-8CE8-8759D8E7D480} = {86937F53-C189-40EF-8CE8-8759D8E7D480} {885D4898-D08D-4091-9C40-C700CFE3FC5A} = {885D4898-D08D-4091-9C40-C700CFE3FC5A} + {947BB5F5-6025-4A4F-8182-1B175469F8D2} = {947BB5F5-6025-4A4F-8182-1B175469F8D2} {900342D7-516A-4469-B1AD-59A66E49A25F} = {900342D7-516A-4469-B1AD-59A66E49A25F} {9E48B300-37D1-11DD-8C41-005056C00008} = {9E48B300-37D1-11DD-8C41-005056C00008} {9EC7190A-249F-4180-A900-548FDCF3055F} = {9EC7190A-249F-4180-A900-548FDCF3055F} @@ -104,6 +105,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_multiprocessing", "_multip EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python3dll", "python3dll.vcxproj", "{885D4898-D08D-4091-9C40-C700CFE3FC5A}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "python3tdll", "python3tdll.vcxproj", "{947BB5F5-6025-4A4F-8182-1B175469F8D2}" +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xxlimited", "xxlimited.vcxproj", "{F749B822-B489-4CA5-A3AD-CE078F5F338A}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testbuffer", "_testbuffer.vcxproj", "{A2697BD3-28C1-4AEC-9106-8B748639FD16}" @@ -168,6 +171,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_remote_debugging", "_remot EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_zstd", "_zstd.vcxproj", "{07029B86-F3E9-443E-86FB-78AA6D47FED1}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xxlimited_35", "xxlimited_35.vcxproj", "{FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM @@ -984,6 +989,38 @@ Global {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|Win32.Build.0 = Release|Win32 {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.ActiveCfg = Release|x64 {885D4898-D08D-4091-9C40-C700CFE3FC5A}.Release|x64.Build.0 = Release|x64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Debug|ARM.ActiveCfg = Debug|ARM + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Debug|ARM.Build.0 = Debug|ARM + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Debug|ARM64.Build.0 = Debug|ARM64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Debug|Win32.ActiveCfg = Debug|Win32 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Debug|Win32.Build.0 = Debug|Win32 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Debug|x64.ActiveCfg = Debug|x64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Debug|x64.Build.0 = Debug|x64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGInstrument|Win32.ActiveCfg = Debug|Win32 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGInstrument|Win32.Build.0 = Debug|Win32 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGInstrument|x64.ActiveCfg = Debug|x64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGInstrument|x64.Build.0 = Debug|x64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGUpdate|Win32.ActiveCfg = Debug|Win32 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGUpdate|Win32.Build.0 = Debug|Win32 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGUpdate|x64.ActiveCfg = Debug|x64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.PGUpdate|x64.Build.0 = Debug|x64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Release|ARM.ActiveCfg = Release|ARM + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Release|ARM.Build.0 = Release|ARM + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Release|ARM64.ActiveCfg = Release|ARM64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Release|ARM64.Build.0 = Release|ARM64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Release|Win32.ActiveCfg = Release|Win32 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Release|Win32.Build.0 = Release|Win32 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Release|x64.ActiveCfg = Release|x64 + {947BB5F5-6025-4A4F-8182-1B175469F8D2}.Release|x64.Build.0 = Release|x64 {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|ARM.ActiveCfg = Debug|ARM {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|ARM64.ActiveCfg = Debug|ARM64 {F749B822-B489-4CA5-A3AD-CE078F5F338A}.Debug|Win32.ActiveCfg = Release|Win32 @@ -1785,6 +1822,38 @@ Global {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|Win32.Build.0 = Release|Win32 {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|x64.ActiveCfg = Release|x64 {07029B86-F3E9-443E-86FB-78AA6D47FED1}.Release|x64.Build.0 = Release|x64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Debug|ARM.ActiveCfg = Debug|ARM + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Debug|ARM.Build.0 = Debug|ARM + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Debug|ARM64.Build.0 = Debug|ARM64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Debug|Win32.ActiveCfg = Debug|Win32 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Debug|Win32.Build.0 = Debug|Win32 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Debug|x64.ActiveCfg = Debug|x64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Debug|x64.Build.0 = Debug|x64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGInstrument|ARM.Build.0 = PGInstrument|ARM + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGInstrument|x64.Build.0 = PGInstrument|x64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGUpdate|ARM.Build.0 = PGUpdate|ARM + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.PGUpdate|x64.Build.0 = PGUpdate|x64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Release|ARM.ActiveCfg = Release|ARM + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Release|ARM.Build.0 = Release|ARM + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Release|ARM64.ActiveCfg = Release|ARM64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Release|ARM64.Build.0 = Release|ARM64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Release|Win32.ActiveCfg = Release|Win32 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Release|Win32.Build.0 = Release|Win32 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Release|x64.ActiveCfg = Release|x64 + {FB868EA7-F93A-4D9B-BE78-CA4E9BA14FFF}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index f79608e1d58dbc8..7435c53e3fdcf9f 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -9,6 +9,7 @@ <OutDir Condition="!HasTrailingSlash($(OutDir))">$(OutDir)\</OutDir> <Py_IntDir Condition="'$(Py_IntDir)' == ''">$(MSBuildThisFileDirectory)obj\</Py_IntDir> <IntDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir> + <IntDir Condition="$(DisableGil) == 'true'">$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)t_$(Configuration)\$(ProjectName)\</IntDir> <IntDir>$(IntDir.Replace(`\\`, `\`))</IntDir> <GeneratedFrozenModulesDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\</GeneratedFrozenModulesDir> <GeneratedZlibNgDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\</GeneratedZlibNgDir> @@ -42,6 +43,10 @@ </PropertyGroup> <PropertyGroup> + <!-- + We conditionally define preprocessor definitions here, and then include all + the properties *unconditionally* in the <PreprocessorDefinition> field below + --> <_DebugPreprocessorDefinition>NDEBUG;</_DebugPreprocessorDefinition> <_DebugPreprocessorDefinition Condition="$(Configuration) == 'Debug'">_DEBUG;</_DebugPreprocessorDefinition> <_PyStatsPreprocessorDefinition>PyStats;</_PyStatsPreprocessorDefinition> @@ -49,15 +54,14 @@ <_PlatformPreprocessorDefinition>_WIN32;</_PlatformPreprocessorDefinition> <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64;</_PlatformPreprocessorDefinition> <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition)</_PlatformPreprocessorDefinition> - <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)";</_Py3NamePreprocessorDefinition> <_FreeThreadedPreprocessorDefinition Condition="$(DisableGil) == 'true'">Py_GIL_DISABLED=1;</_FreeThreadedPreprocessorDefinition> <_PymallocHugepagesPreprocessorDefinition Condition="$(UsePymallocHugepages) == 'true'">PYMALLOC_USE_HUGEPAGES=1;</_PymallocHugepagesPreprocessorDefinition> + <_PyUsingPgoPreprocessorDefinition Condition="'$(SupportPGO)' and ($(Configuration) == 'PGInstrument' or $(Configuration) == 'PGUpdate')">_Py_USING_PGO=1;</_PyUsingPgoPreprocessorDefinition> </PropertyGroup> <ItemDefinitionGroup> <ClCompile> <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <PreprocessorDefinitions>WIN32;$(_Py3NamePreprocessorDefinition)$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)$(_FreeThreadedPreprocessorDefinition)$(_PymallocHugepagesPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions> - <PreprocessorDefinitions Condition="'$(SupportPGO)' and ($(Configuration) == 'PGInstrument' or $(Configuration) == 'PGUpdate')">_Py_USING_PGO=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>WIN32;$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)$(_FreeThreadedPreprocessorDefinition)$(_PymallocHugepagesPreprocessorDefinition)$(_PyUsingPgoPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions> <Optimization>MaxSpeed</Optimization> <IntrinsicFunctions>true</IntrinsicFunctions> diff --git a/PCbuild/python.props b/PCbuild/python.props index f29f3d18de5f9d2..8d931bba28a389a 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -43,7 +43,7 @@ <PySourcePath Condition="'$(PySourcePath)' == ''">$([System.IO.Path]::GetFullPath($(MSBuildThisFileDirectory)\..\))</PySourcePath> <PySourcePath Condition="!HasTrailingSlash($(PySourcePath))">$(PySourcePath)\</PySourcePath> - <!-- Directory where build outputs are put --> + <!-- Directories where build outputs are put --> <BuildPath32 Condition="'$(Py_OutDir)' == ''">$(PySourcePath)PCbuild\win32\</BuildPath32> <BuildPath32 Condition="'$(Py_OutDir)' != ''">$(Py_OutDir)\win32\</BuildPath32> <BuildPath64 Condition="'$(Py_OutDir)' == ''">$(PySourcePath)PCbuild\amd64\</BuildPath64> @@ -52,11 +52,34 @@ <BuildPathArm32 Condition="'$(Py_OutDir)' != ''">$(Py_OutDir)\arm32\</BuildPathArm32> <BuildPathArm64 Condition="'$(Py_OutDir)' == ''">$(PySourcePath)PCbuild\arm64\</BuildPathArm64> <BuildPathArm64 Condition="'$(Py_OutDir)' != ''">$(Py_OutDir)\arm64\</BuildPathArm64> + <BuildPath32t Condition="'$(Py_OutDir)' == ''">$(PySourcePath)PCbuild\win32t\</BuildPath32t> + <BuildPath32t Condition="'$(Py_OutDir)' != ''">$(Py_OutDir)\win32t\</BuildPath32t> + <BuildPath64t Condition="'$(Py_OutDir)' == ''">$(PySourcePath)PCbuild\amd64t\</BuildPath64t> + <BuildPath64t Condition="'$(Py_OutDir)' != ''">$(Py_OutDir)\amd64t\</BuildPath64t> + <BuildPathArm32t Condition="'$(Py_OutDir)' == ''">$(PySourcePath)PCbuild\arm32t\</BuildPathArm32t> + <BuildPathArm32t Condition="'$(Py_OutDir)' != ''">$(Py_OutDir)\arm32t\</BuildPathArm32t> + <BuildPathArm64t Condition="'$(Py_OutDir)' == ''">$(PySourcePath)PCbuild\arm64t\</BuildPathArm64t> + <BuildPathArm64t Condition="'$(Py_OutDir)' != ''">$(Py_OutDir)\arm64t\</BuildPathArm64t> + </PropertyGroup> + + <PropertyGroup Condition="$(DisableGil) != 'true'"> <BuildPath Condition="'$(ArchName)' == 'win32'">$(BuildPath32)</BuildPath> <BuildPath Condition="'$(ArchName)' == 'amd64'">$(BuildPath64)</BuildPath> <BuildPath Condition="'$(ArchName)' == 'arm32'">$(BuildPathArm32)</BuildPath> <BuildPath Condition="'$(ArchName)' == 'arm64'">$(BuildPathArm64)</BuildPath> <BuildPath Condition="'$(BuildPath)' == ''">$(PySourcePath)PCbuild\$(ArchName)\</BuildPath> + </PropertyGroup> + + <PropertyGroup Condition="$(DisableGil) == 'true'"> + <!-- Directory where build outputs are put --> + <BuildPath Condition="'$(ArchName)' == 'win32'">$(BuildPath32t)</BuildPath> + <BuildPath Condition="'$(ArchName)' == 'amd64'">$(BuildPath64t)</BuildPath> + <BuildPath Condition="'$(ArchName)' == 'arm32'">$(BuildPathArm32t)</BuildPath> + <BuildPath Condition="'$(ArchName)' == 'arm64'">$(BuildPathArm64t)</BuildPath> + <BuildPath Condition="'$(BuildPath)' == ''">$(PySourcePath)PCbuild\$(ArchName)t\</BuildPath> + </PropertyGroup> + + <PropertyGroup> <BuildPath Condition="!HasTrailingSlash($(BuildPath))">$(BuildPath)\</BuildPath> <BuildPath Condition="$(Configuration) == 'PGInstrument'">$(BuildPath)instrumented\</BuildPath> @@ -75,15 +98,15 @@ <Import Project="$(ExternalProps)" Condition="$(ExternalProps) != '' and Exists('$(ExternalProps)')" /> <PropertyGroup> - <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.50.4.0\</sqlite3Dir> + <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.53.2.0\</sqlite3Dir> <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir> <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.8.1.1\</lzmaDir> <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir> <libffiOutDir Condition="$(libffiOutDir) == ''">$(libffiDir)$(ArchName)\</libffiOutDir> <libffiIncludeDir Condition="$(libffiIncludeDir) == ''">$(libffiOutDir)include</libffiIncludeDir> <mpdecimalDir Condition="$(mpdecimalDir) == ''">$(ExternalsDir)\mpdecimal-4.0.0\</mpdecimalDir> - <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.5.6\</opensslDir> - <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.5.6\$(ArchName)\</opensslOutDir> + <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.5.7\</opensslDir> + <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.5.7\$(ArchName)\</opensslOutDir> <opensslIncludeDir Condition="$(opensslIncludeDir) == ''">$(opensslOutDir)include</opensslIncludeDir> <nasmDir Condition="$(nasmDir) == ''">$(ExternalsDir)\nasm-2.11.06\</nasmDir> <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.3.1\</zlibDir> @@ -232,18 +255,17 @@ <Field3Value Condition="$(UseTestMarker) == 'true'">$([msbuild]::Add($(Field3Value), 9000))</Field3Value> <!-- Name and full path of the resulting python.exe binary --> - <PyExeName Condition="$(DisableGil) == 'true'">python$(MajorVersionNumber).$(MinorVersionNumber)t</PyExeName> <PyExeName Condition="$(PyExeName) == ''">python</PyExeName> <PythonExe Condition="'$(PythonExe)' == ''">$(BuildPath)$(PyExeName)$(PyDebugExt).exe</PythonExe> - <PyWExeName Condition="$(DisableGil) == 'true'">pythonw$(MajorVersionNumber).$(MinorVersionNumber)t</PyWExeName> <PyWExeName Condition="$(PyWExeName) == ''">pythonw</PyWExeName> <!-- The name of the resulting pythonXY.dll (without the extension) --> <PyDllName Condition="$(DisableGil) == 'true'">python$(MajorVersionNumber)$(MinorVersionNumber)t$(PyDebugExt)</PyDllName> <PyDllName Condition="$(PyDllName) == ''">python$(MajorVersionNumber)$(MinorVersionNumber)$(PyDebugExt)</PyDllName> + <!-- The name of the resulting pythonX.dll (without the extension) --> - <Py3DllName Condition="$(DisableGil) == 'true'">python3t</Py3DllName> <Py3DllName Condition="$(Py3DllName) == ''">python3</Py3DllName> + <Abi3tDllName Condition="$(Abi3tDllName) == ''">python3t</Abi3tDllName> <!-- The version and platform tag to include in .pyd filenames --> <PydTag Condition="$(ArchName) == 'win32'">.cp$(MajorVersionNumber)$(MinorVersionNumber)-win32</PydTag> diff --git a/PCbuild/python.vcxproj b/PCbuild/python.vcxproj index 70dabaa3c8bc027..417ede34c54af3a 100644 --- a/PCbuild/python.vcxproj +++ b/PCbuild/python.vcxproj @@ -135,6 +135,14 @@ set PYTHONPATH=$(PySourcePath)Lib "$(OutDir)$(PyExeName)$(PyDebugExt).exe" "$(PySourcePath)PC\validate_ucrtbase.py" $(UcrtName)' ContinueOnError="true" /> </Target> + <Target Name="CopyFreethreadedBinary" AfterTargets="AfterBuild" + Condition="$(DisableGil) == 'true' and $(Configuration) != 'PGInstrument'"> + <Message Text="Duplicating $(TargetPath) to $(PyExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe for free-threaded compatibility" /> + <Copy SourceFiles="$(TargetPath)" + DestinationFiles="$(OutDir)\$(PyExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe" + SkipUnchangedFiles="true" + UseHardLinksIfPossible="true" /> + </Target> <Target Name="GeneratePythonBat" AfterTargets="AfterBuild"> <PropertyGroup> <_Content>@rem This script invokes the most recently built Python with all arguments diff --git a/PCbuild/python3dll.vcxproj b/PCbuild/python3dll.vcxproj index 235ea1cf9d33fb9..3d8ac1b23532c1d 100644 --- a/PCbuild/python3dll.vcxproj +++ b/PCbuild/python3dll.vcxproj @@ -75,7 +75,7 @@ <Import Project="python.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> <PropertyGroup Label="Configuration"> - <TargetName>$(Py3DllName)</TargetName> + <TargetName>python3</TargetName> <ConfigurationType>DynamicLibrary</ConfigurationType> </PropertyGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> diff --git a/PCbuild/python3tdll.vcxproj b/PCbuild/python3tdll.vcxproj new file mode 100644 index 000000000000000..796712cca3146c4 --- /dev/null +++ b/PCbuild/python3tdll.vcxproj @@ -0,0 +1,110 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|ARM"> + <Configuration>Debug</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|ARM64"> + <Configuration>Debug</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|Win32"> + <Configuration>Debug</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|ARM"> + <Configuration>PGInstrument</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|ARM64"> + <Configuration>PGInstrument</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|Win32"> + <Configuration>PGInstrument</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|x64"> + <Configuration>PGInstrument</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|ARM"> + <Configuration>PGUpdate</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|ARM64"> + <Configuration>PGUpdate</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|Win32"> + <Configuration>PGUpdate</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|x64"> + <Configuration>PGUpdate</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|ARM"> + <Configuration>Release</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|ARM64"> + <Configuration>Release</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|Win32"> + <Configuration>Release</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <ProjectGuid>{947BB5F5-6025-4A4F-8182-1B175469F8D2}</ProjectGuid> + <RootNamespace>python3tdll</RootNamespace> + <Keyword>Win32Proj</Keyword> + <SupportPGO>false</SupportPGO> + </PropertyGroup> + <Import Project="python.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Label="Configuration"> + <TargetName>python3t</TargetName> + <ConfigurationType>DynamicLibrary</ConfigurationType> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="pyproject.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup> + <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion> + </PropertyGroup> + <ItemDefinitionGroup> + <ClCompile> + <PreprocessorDefinitions>PYTHON_DLL_NAME="$(PyDllName)";%(PreprocessorDefinitions)</PreprocessorDefinitions> + <BufferSecurityCheck>false</BufferSecurityCheck> + </ClCompile> + <Link> + <NoEntryPoint>true</NoEntryPoint> + </Link> + </ItemDefinitionGroup> + <ItemGroup> + <ClCompile Include="..\PC\python3dll.c" /> + </ItemGroup> + <ItemGroup> + <ResourceCompile Include="..\PC\python_nt.rc" /> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + </ImportGroup> +</Project> diff --git a/PCbuild/python3tdll.vcxproj.filters b/PCbuild/python3tdll.vcxproj.filters new file mode 100644 index 000000000000000..37510e3c7398f28 --- /dev/null +++ b/PCbuild/python3tdll.vcxproj.filters @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <Filter Include="Source Files"> + <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier> + <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions> + </Filter> + <Filter Include="Resource Files"> + <UniqueIdentifier>{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}</UniqueIdentifier> + <Extensions>rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav</Extensions> + </Filter> + </ItemGroup> + <ItemGroup> + <ClCompile Include="..\PC\python3dll.c"> + <Filter>Source Files</Filter> + </ClCompile> + </ItemGroup> + <ItemGroup> + <ResourceCompile Include="..\PC\python_nt.rc"> + <Filter>Resource Files</Filter> + </ResourceCompile> + </ItemGroup> +</Project> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index fae4a90b4536fcb..e255ed5af19125d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -316,6 +316,8 @@ <ClInclude Include="..\Include\internal\pycore_setobject.h" /> <ClInclude Include="..\Include\internal\pycore_signal.h" /> <ClInclude Include="..\Include\internal\pycore_sliceobject.h" /> + <ClInclude Include="..\Include\internal\pycore_slots.h" /> + <ClInclude Include="..\Include\internal\pycore_slots_generated.h" /> <ClInclude Include="..\Include\internal\pycore_stackref.h" /> <ClInclude Include="..\Include\internal\pycore_stats.h" /> <ClInclude Include="..\Include\internal\pycore_strhex.h" /> @@ -387,6 +389,8 @@ <ClInclude Include="..\Include\refcount.h" /> <ClInclude Include="..\Include\setobject.h" /> <ClInclude Include="..\Include\sliceobject.h" /> + <ClInclude Include="..\Include\slots.h" /> + <ClInclude Include="..\Include\slots_generated.h" /> <ClInclude Include="..\Include\structmember.h" /> <ClInclude Include="..\Include\structseq.h" /> <ClInclude Include="..\Include\sysmodule.h" /> @@ -618,7 +622,10 @@ <ClCompile Include="..\Python\critical_section.c" /> <ClCompile Include="..\Python\crossinterp.c" /> <ClCompile Include="..\Python\dynamic_annotations.c" /> - <ClCompile Include="..\Python\dynload_win.c" /> + <ClCompile Include="..\Python\dynload_win.c"> + <PreprocessorDefinitions Condition="$(DisableGil) != 'true'">PY3_DLLNAME=L"$(Py3DllName)";%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>ABI3T_DLLNAME=L"$(Abi3tDllName)";%(PreprocessorDefinitions)</PreprocessorDefinitions> + </ClCompile> <ClCompile Include="..\Python\errors.c" /> <ClCompile Include="..\Python\fileutils.c" /> <ClCompile Include="..\Python\flowgraph.c" /> @@ -648,6 +655,7 @@ <ClCompile Include="..\Python\instruction_sequence.c" /> <ClCompile Include="..\Python\instrumentation.c" /> <ClCompile Include="..\Python\jit.c" /> + <ClCompile Include="..\Python\jit_publish.c" /> <ClCompile Include="..\Python\legacy_tracing.c" /> <ClCompile Include="..\Python\lock.c" /> <ClCompile Include="..\Python\marshal.c" /> @@ -682,6 +690,8 @@ <ClCompile Include="..\Python\pythonrun.c" /> <ClCompile Include="..\Python\specialize.c" /> <ClCompile Include="..\Python\suggestions.c" /> + <ClCompile Include="..\Python\slots.c" /> + <ClCompile Include="..\Python\slots_generated.c" /> <ClCompile Include="..\Python\structmember.c" /> <ClCompile Include="..\Python\symtable.c" /> <ClCompile Include="..\Python\sysmodule.c"> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 04b6641ae30e7f8..649ee1859ff9961 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -228,6 +228,12 @@ <ClInclude Include="..\Include\sliceobject.h"> <Filter>Include</Filter> </ClInclude> + <ClInclude Include="..\Include\slots.h"> + <Filter>Include</Filter> + </ClInclude> + <ClInclude Include="..\Include\slots_generated.h"> + <Filter>Include</Filter> + </ClInclude> <ClInclude Include="..\Include\stats.h"> <Filter>Include</Filter> </ClInclude> @@ -846,6 +852,12 @@ <ClInclude Include="..\Include\internal\pycore_sliceobject.h"> <Filter>Include\internal</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_slots.h"> + <Filter>Include\internal</Filter> + </ClInclude> + <ClInclude Include="..\Include\internal\pycore_slots_generated.h"> + <Filter>Include\internal</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_strhex.h"> <Filter>Include\internal</Filter> </ClInclude> @@ -1481,6 +1493,9 @@ <ClCompile Include="..\Python\jit.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Python\jit_publish.c"> + <Filter>Python</Filter> + </ClCompile> <ClCompile Include="..\Python\legacy_tracing.c"> <Filter>Source Files</Filter> </ClCompile> @@ -1574,6 +1589,12 @@ <ClCompile Include="..\Python\specialize.c"> <Filter>Python</Filter> </ClCompile> + <ClCompile Include="..\Objects\slots.c"> + <Filter>Objects</Filter> + </ClCompile> + <ClCompile Include="..\Objects\slots_generated.c"> + <Filter>Objects</Filter> + </ClCompile> <ClCompile Include="..\Python\structmember.c"> <Filter>Python</Filter> </ClCompile> diff --git a/PCbuild/pythonw.vcxproj b/PCbuild/pythonw.vcxproj index c6a5b8ce90a0d9b..244cdf622ad915c 100644 --- a/PCbuild/pythonw.vcxproj +++ b/PCbuild/pythonw.vcxproj @@ -115,4 +115,12 @@ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> </ImportGroup> + <Target Name="CopyFreethreadedBinary" AfterTargets="AfterBuild" + Condition="$(DisableGil) == 'true' and $(Configuration) != 'PGInstrument'"> + <Message Text="Duplicating $(TargetPath) to $(PyWExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe for free-threaded compatibility" /> + <Copy SourceFiles="$(TargetPath)" + DestinationFiles="$(OutDir)\$(PyWExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe" + SkipUnchangedFiles="true" + UseHardLinksIfPossible="true" /> + </Target> </Project> \ No newline at end of file diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index b98a956034c5375..7c5eab2eb75a1de 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -160,12 +160,17 @@ pyshellext pyshellext.dll, the shell extension deployed with the launcher python3dll python3.dll, the PEP 384 Stable ABI dll + (not installed on free-threaded builds) +python3tdll + python3t.dll, the PEP 803 free-threading Stable ABI dll + (built from the same source as python3.dll) xxlimited builds an example module that makes use of the PEP 384 Stable ABI, see Modules\xxlimited.c xxlimited_35 - ditto for testing the Python 3.5 stable ABI, see - Modules\xxlimited_35.c +xxlimited_3_13 + ditto for testing older Limited API, see + Modules\xxlimited_*.c The following sub-projects are for individual modules of the standard library which are implemented in C; each one builds a DLL (renamed to @@ -237,12 +242,12 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.50.4, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.53.2, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter - Wraps version 8.6.15 of the Tk windowing system, which is downloaded + Wraps version 9.0.3 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index f1e0607393405be..d5c9a24f292327c 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -32,6 +32,7 @@ setlocal set pcbuild=%~dp0 set pyname=python set suffix= +set suffix1= set qmode= set dashO= set regrtestargs=--fast-ci @@ -41,8 +42,7 @@ set exe= if "%~1"=="-O" (set dashO=-O) & shift & goto CheckOpts if "%~1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%~1"=="-d" (set suffix=_d) & shift & goto CheckOpts -rem HACK: Need some way to infer the version number in this script -if "%~1"=="--disable-gil" (set pyname=python3.15t) & shift & goto CheckOpts +if "%~1"=="--disable-gil" (set suffix1=t) & shift & goto CheckOpts if "%~1"=="-win32" (set prefix=%pcbuild%win32) & shift & goto CheckOpts if "%~1"=="-x64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts if "%~1"=="-amd64" (set prefix=%pcbuild%amd64) & shift & goto CheckOpts @@ -52,7 +52,7 @@ if "%~1"=="-p" (call :SetPlatform %~2) & shift & shift & goto CheckOpts if NOT "%~1"=="" (set regrtestargs=%regrtestargs% %~1) & shift & goto CheckOpts if not defined prefix set prefix=%pcbuild%amd64 -set exe=%prefix%\%pyname%%suffix%.exe +set exe=%prefix%%suffix1%\%pyname%%suffix%.exe set cmd="%exe%" %dashO% -m test %regrtestargs% if defined qmode goto Qmode @@ -60,7 +60,7 @@ echo Deleting .pyc files ... "%exe%" "%pcbuild%rmpyc.py" echo Cleaning _pth files ... -if exist %prefix%\*._pth del %prefix%\*._pth +if exist %prefix%%suffix1%\*._pth del %prefix%%suffix1%\*._pth echo on %cmd% diff --git a/PCbuild/tcl.vcxproj b/PCbuild/tcl.vcxproj index ab68db9210fbe40..2233559dfc8f5a1 100644 --- a/PCbuild/tcl.vcxproj +++ b/PCbuild/tcl.vcxproj @@ -63,8 +63,8 @@ <NMakeBuildCommandLine>setlocal set VCINSTALLDIR=$(VCInstallDir) cd /D "$(tclDir)win" -nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) $(WarningsFlags) TCLSH_NATIVE="$(tclWin32Exe)" core shell dlls -nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) $(WarningsFlags) TCLSH_NATIVE="$(tclWin32Exe)" install-binaries install-libraries +nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) $(WarningsFlags) $(TclshNativeFlag) core shell dlls +nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) $(WarningsFlags) $(TclshNativeFlag) install-binaries install-libraries copy /Y ..\license.terms "$(OutDir)\tcllicense.terms" </NMakeBuildCommandLine> </PropertyGroup> @@ -74,4 +74,4 @@ copy /Y ..\license.terms "$(OutDir)\tcllicense.terms" <Target Name="Clean" /> <Target Name="ResolveAssemblyReferences" /> -</Project> \ No newline at end of file +</Project> diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index d26b36ba98e493d..28e8c0db4d1eafd 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> <Import Project="pyproject.props" Condition="$(__PyProject_Props_Imported) != 'true'" /> <PropertyGroup> - <TclVersion Condition="$(TclVersion) == ''">8.6.15.0</TclVersion> + <TclVersion Condition="$(TclVersion) == ''">9.0.3.0</TclVersion> <TkVersion Condition="$(TkVersion) == ''">$(TclVersion)</TkVersion> <TclMajorVersion>$([System.Version]::Parse($(TclVersion)).Major)</TclMajorVersion> <TclMinorVersion>$([System.Version]::Parse($(TclVersion)).Minor)</TclMinorVersion> @@ -12,16 +12,17 @@ <TkMinorVersion>$([System.Version]::Parse($(TkVersion)).Minor)</TkMinorVersion> <TkPatchLevel>$([System.Version]::Parse($(TkVersion)).Build)</TkPatchLevel> <TkRevision>$([System.Version]::Parse($(TkVersion)).Revision)</TkRevision> - <tclDir Condition="$(tclDir) == ''">$(ExternalsDir)tcl-core-$(TclVersion)\</tclDir> + <!-- Back compat hack. If you're here wondering about -core vs not, do us both a favor and just override the whole tclDir variable --> + <tclDir Condition="$(tclDir) == '' and $(TclMajorVersion) == '8'">$(ExternalsDir)tcl-core-$(TclVersion)\</tclDir> + <tclDir Condition="$(tclDir) == ''">$(ExternalsDir)tcl-$(TclVersion)\</tclDir> <tkDir Condition="$(tkDir) == ''">$(ExternalsDir)tk-$(TkVersion)\</tkDir> <tcltkDir Condition="$(tcltkDir) == ''">$(ExternalsDir)tcltk-$(TclVersion)\$(ArchName)\</tcltkDir> - <tclWin32Exe Condition="$(Platform) == 'Win32'">$(tcltkDir)\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)t.exe</tclWin32Exe> - <tclWin32Exe Condition="$(Platform) != 'Win32'">$(tcltkDir)\..\win32\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)t.exe</tclWin32Exe> + <tcltkSuffix Condition="'$(TclMajorVersion)' == '8'">t</tcltkSuffix> + <tkPrefix Condition="'$(TclMajorVersion)' == '9'">tcl9</tkPrefix> + <TclshNativeFlag Condition="$(Platform) != 'Win32'">TCLSH_NATIVE="$(tcltkDir)\..\win32\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix).exe"</TclshNativeFlag> <tclExternalTommath Condition="$(TclMajorVersion) == '9'">TCL_WITH_EXTERNAL_TOMMATH;</tclExternalTommath> <!--<TclDebugExt Condition="'$(Configuration)' == 'Debug'">g</TclDebugExt>--> - <tcltkSuffix Condition="'$(TclMajorVersion)' == '8'">t</tcltkSuffix> - <tkPrefix Condition="'$(TclMajorVersion)' == '9'">tcl9</tkPrefix> <tclDLLName >tcl$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).dll</tclDLLName> <tclLibName>tcl$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).lib</tclLibName> <tclShExeName>tclsh$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix)$(TclDebugExt).exe</tclShExeName> diff --git a/PCbuild/tk.vcxproj b/PCbuild/tk.vcxproj index b111969ca5de6c7..204244db0d3e4bb 100644 --- a/PCbuild/tk.vcxproj +++ b/PCbuild/tk.vcxproj @@ -64,8 +64,8 @@ <NMakeBuildCommandLine>setlocal set VCINSTALLDIR=$(VCInstallDir) cd /D "$(tkDir)win" -nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) $(WarningsFlags) TCLSH_NATIVE="$(tclWin32Exe)" all -nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) $(WarningsFlags) TCLSH_NATIVE="$(tclWin32Exe)" install-binaries install-libraries +nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) $(WarningsFlags) $(TclshNativeFlag) all +nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) $(WarningsFlags) $(TclshNativeFlag) install-binaries install-libraries copy /Y ..\license.terms "$(OutDir)\tklicense.terms" </NMakeBuildCommandLine> </PropertyGroup> @@ -80,4 +80,4 @@ copy /Y ..\license.terms "$(OutDir)\tklicense.terms" <Target Name="Clean" /> <Target Name="ResolveAssemblyReferences" /> -</Project> \ No newline at end of file +</Project> diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index abaf3a979af2681..a2e8ffa82b10eb7 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -89,10 +89,13 @@ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> <Import Project="pyproject.props" /> </ImportGroup> - <PropertyGroup Label="UserMacros" /> + <PropertyGroup Label="UserMacros"> + <ExeName>$(PyExeName)$(PyDebugExt).exe</ExeName> + <ExeName Condition="$(DisableGil) == 'true'">$(PyExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe</ExeName> + </PropertyGroup> <ItemDefinitionGroup> <ClCompile> - <PreprocessorDefinitions>EXENAME=L"$(PyExeName)$(PyDebugExt).exe";_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>EXENAME=L"$(ExeName)";_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <ResourceCompile> diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index c58280deb8abeb3..f2aaf83fe2b3785 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -89,10 +89,13 @@ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> <Import Project="pyproject.props" /> </ImportGroup> - <PropertyGroup Label="UserMacros" /> + <PropertyGroup Label="UserMacros"> + <ExeName>$(PyWExeName)$(PyDebugExt).exe</ExeName> + <ExeName Condition="$(DisableGil) == 'true'">$(PyWExeName)$(MajorVersionNumber).$(MinorVersionNumber)t$(PyDebugExt).exe</ExeName> + </PropertyGroup> <ItemDefinitionGroup> <ClCompile> - <PreprocessorDefinitions>EXENAME=L"$(PyWExeName)$(PyDebugExt).exe";_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions>EXENAME=L"$(ExeName)";_WINDOWS;%(PreprocessorDefinitions)</PreprocessorDefinitions> <RuntimeLibrary>MultiThreaded</RuntimeLibrary> </ClCompile> <ResourceCompile> diff --git a/PCbuild/xxlimited.vcxproj b/PCbuild/xxlimited.vcxproj index 093e6920c0b76ce..f0c3616600148f1 100644 --- a/PCbuild/xxlimited.vcxproj +++ b/PCbuild/xxlimited.vcxproj @@ -104,6 +104,9 @@ <ProjectReference Include="python3dll.vcxproj"> <Project>{885d4898-d08d-4091-9c40-c700cfe3fc5a}</Project> </ProjectReference> + <ProjectReference Include="python3tdll.vcxproj"> + <Project>{947BB5F5-6025-4A4F-8182-1B175469F8D2}</Project> + </ProjectReference> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> diff --git a/PCbuild/xxlimited_35.vcxproj b/PCbuild/xxlimited_35.vcxproj index 3f4d4463f24af07..bfaf4e253664d4b 100644 --- a/PCbuild/xxlimited_35.vcxproj +++ b/PCbuild/xxlimited_35.vcxproj @@ -104,6 +104,9 @@ <ProjectReference Include="python3dll.vcxproj"> <Project>{885d4898-d08d-4091-9c40-c700cfe3fc5a}</Project> </ProjectReference> + <ProjectReference Include="python3tdll.vcxproj"> + <Project>{947BB5F5-6025-4A4F-8182-1B175469F8D2}</Project> + </ProjectReference> </ItemGroup> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <ImportGroup Label="ExtensionTargets"> diff --git a/PCbuild/xxlimited_3_13.vcxproj b/PCbuild/xxlimited_3_13.vcxproj new file mode 100644 index 000000000000000..7a9760fd43121ef --- /dev/null +++ b/PCbuild/xxlimited_3_13.vcxproj @@ -0,0 +1,111 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|ARM"> + <Configuration>Debug</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|ARM64"> + <Configuration>Debug</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|Win32"> + <Configuration>Debug</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|ARM"> + <Configuration>PGInstrument</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|ARM64"> + <Configuration>PGInstrument</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|Win32"> + <Configuration>PGInstrument</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGInstrument|x64"> + <Configuration>PGInstrument</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|ARM"> + <Configuration>PGUpdate</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|ARM64"> + <Configuration>PGUpdate</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|Win32"> + <Configuration>PGUpdate</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="PGUpdate|x64"> + <Configuration>PGUpdate</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|ARM"> + <Configuration>Release</Configuration> + <Platform>ARM</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|ARM64"> + <Configuration>Release</Configuration> + <Platform>ARM64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|Win32"> + <Configuration>Release</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <ProjectGuid>{fb868ea7-f93a-4d9b-be78-ca4e9ba14fff}</ProjectGuid> + <RootNamespace>xxlimited_3_13</RootNamespace> + <Keyword>Win32Proj</Keyword> + </PropertyGroup> + <Import Project="python.props" /> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Label="Configuration"> + <ConfigurationType>DynamicLibrary</ConfigurationType> + <CharacterSet>NotSet</CharacterSet> + <SupportPGO>false</SupportPGO> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <PropertyGroup> + <TargetExt>$(PyStdlibPydExt)</TargetExt> + </PropertyGroup> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="pyproject.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup> + <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion> + </PropertyGroup> + <ItemDefinitionGroup> + <Link> + <AdditionalDependencies>wsock32.lib;%(AdditionalDependencies)</AdditionalDependencies> + </Link> + </ItemDefinitionGroup> + <ItemGroup> + <ClCompile Include="..\Modules\xxlimited_3_13.c" /> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="python3dll.vcxproj"> + <Project>{885d4898-d08d-4091-9c40-c700cfe3fc5a}</Project> + </ProjectReference> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + </ImportGroup> +</Project> diff --git a/PCbuild/xxlimited_3_13.vcxproj.filters b/PCbuild/xxlimited_3_13.vcxproj.filters new file mode 100644 index 000000000000000..3dfb7800edc4419 --- /dev/null +++ b/PCbuild/xxlimited_3_13.vcxproj.filters @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <Filter Include="Source Files"> + <UniqueIdentifier>{5be27194-6530-452d-8d86-3767b991fa83}</UniqueIdentifier> + </Filter> + </ItemGroup> + <ItemGroup> + <ClCompile Include="..\Modules\xxlimited_3_13.c"> + <Filter>Source Files</Filter> + </ClCompile> + </ItemGroup> +</Project> diff --git a/PCbuild/zlib-ng.vcxproj b/PCbuild/zlib-ng.vcxproj index de1698ae718473a..ffe8e70f2dbbc7d 100644 --- a/PCbuild/zlib-ng.vcxproj +++ b/PCbuild/zlib-ng.vcxproj @@ -219,13 +219,15 @@ <PropertyGroup> <Text>$([System.IO.File]::ReadAllText('$(zlibNgDir)\zlib.h.in').Replace('@ZLIB_SYMBOL_PREFIX@', ''))</Text> </PropertyGroup> - <WriteLinesToFile File="$(IntDir)zlib.h" Lines="$(Text)" /> + <MakeDir Directories="$(GeneratedZlibNgDir)" /> + <WriteLinesToFile File="$(GeneratedZlibNgDir)zlib.h" Lines="$(Text)" /> </Target> <Target Name="_EnsureZlibNgH" Inputs="$(zlibNgDir)\zlib-ng.h.in" Outputs="$(IntDir)zlib-ng.h"> <PropertyGroup> <Text>$([System.IO.File]::ReadAllText('$(zlibNgDir)\zlib-ng.h.in').Replace('@ZLIB_SYMBOL_PREFIX@', ''))</Text> </PropertyGroup> - <WriteLinesToFile File="$(IntDir)zlib-ng.h" Lines="$(Text)" /> + <MakeDir Directories="$(GeneratedZlibNgDir)" /> + <WriteLinesToFile File="$(GeneratedZlibNgDir)zlib-ng.h" Lines="$(Text)" /> </Target> <Target Name="_EnsureZlibNgHeaders" BeforeTargets="PrepareForBuild" diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 5e52bb838719046..d5315c07f13bb13 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1295,24 +1295,146 @@ _PyPegen_nonparen_genexp_in_call(Parser *p, expr_ty args, asdl_comprehension_seq // Fstring stuff +static int +unicodewriter_write_line(Parser *p, PyUnicodeWriter *w, const char *line_start, const char *line_end, + int is_raw, Token* token) +{ + if (is_raw || memchr(line_start, '\\', line_end - line_start) == NULL) { + return PyUnicodeWriter_WriteUTF8(w, line_start, line_end - line_start); + } + else { + PyObject *line = _PyPegen_decode_string(p, 1, line_start, line_end - line_start, token); + if (line == NULL || PyUnicodeWriter_WriteStr(w, line) < 0) { + Py_XDECREF(line); + return -1; + } + Py_DECREF(line); + } + return 0; +} + +static PyObject* +_PyPegen_dedent_string_part( + Parser *p, const char *s, size_t len, const char *indent, Py_ssize_t indent_len, + int is_first, int is_raw, expr_ty constant, Token* token) +{ + Py_ssize_t lineno = constant->lineno; + const char *line_start = s; + const char *end = s + len; + + int _prev_call_invalid = p->call_invalid_rules; + if (!_prev_call_invalid && !is_raw) { + // _PyPegen_decode_string() and decode_bytes_with_escapes() may call + // warn_invalid_escape_sequence(). It may emit issue or raise SyntaxError + // for invalid escape sequences. + // We need to call it before dedenting since SyntaxError needs exact lineno + // and col_offset of invalid escape sequences. + PyObject *t = _PyPegen_decode_string(p, 0, s, len, token); + if (t == NULL) { + return NULL; + } + Py_DECREF(t); + p->call_invalid_rules = 1; + } + + PyUnicodeWriter *w = PyUnicodeWriter_Create(len); + if (w == NULL) { + return NULL; + } + + if (is_first) { + assert (line_start[0] == '\n'); + line_start++; // skip the first newline + } + else { + // Example: df""" + // first part {param} second part + // next line + // """" + // We don't need to dedent the first line in the non-first parts. + const char *line_end = memchr(line_start, '\n', end - line_start); + if (line_end) { + line_end++; // include the newline + } + else { + line_end = end; + } + if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { + goto error; + } + line_start = line_end; + } + + while (line_start < end) { + lineno++; + + Py_ssize_t i = 0; + while (line_start + i < end && i < indent_len && line_start[i] == indent[i]) { + i++; + } + + if (line_start[i] == '\0') { // found an empty line without newline. + break; + } + if (line_start[i] == '\n') { // found an empty line with newline. + if (PyUnicodeWriter_WriteChar(w, '\n') < 0) { + goto error; + } + line_start += i+1; + continue; + } + if (i < indent_len) { // found an invalid indent. + assert(line_start[i] != indent[i]); + RAISE_ERROR_KNOWN_LOCATION(p, PyExc_SyntaxError, lineno, i, lineno, i+1, + "d-string line missing valid indentation"); + goto error; + } + + // found a indented line. let's dedent it. + line_start += i; + const char *line_end = memchr(line_start, '\n', end - line_start); + if (line_end) { + line_end++; // include the newline + } + else { + line_end = end; + } + if (unicodewriter_write_line(p, w, line_start, line_end, is_raw, token) < 0) { + goto error; + } + line_start = line_end; + } + p->call_invalid_rules = _prev_call_invalid; + return PyUnicodeWriter_Finish(w); + +error: + p->call_invalid_rules = _prev_call_invalid; + PyUnicodeWriter_Discard(w); + return NULL; +} + static expr_ty -_PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* token) { +_PyPegen_decode_fstring_part(Parser* p, int is_first, int is_raw, + const char *indent, Py_ssize_t indent_len, + expr_ty constant, Token* token) +{ assert(PyUnicode_CheckExact(constant->v.Constant.value)); const char* bstr = PyUnicode_AsUTF8(constant->v.Constant.value); if (bstr == NULL) { return NULL; } + is_raw = is_raw || strchr(bstr, '\\') == NULL; - size_t len; - if (strcmp(bstr, "{{") == 0 || strcmp(bstr, "}}") == 0) { - len = 1; - } else { - len = strlen(bstr); + PyObject *str = NULL; + if (indent != NULL) { + str = _PyPegen_dedent_string_part(p, bstr, strlen(bstr), indent, indent_len, + is_first, is_raw, constant, token); + } + else { + str = _PyPegen_decode_string(p, is_raw, bstr, strlen(bstr), token); } - is_raw = is_raw || strchr(bstr, '\\') == NULL; - PyObject *str = _PyPegen_decode_string(p, is_raw, bstr, len, token); if (str == NULL) { _Pypegen_raise_decode_error(p); return NULL; @@ -1326,6 +1448,103 @@ _PyPegen_decode_fstring_part(Parser* p, int is_raw, expr_ty constant, Token* tok p->arena); } +/* +This function is customized version of _Py_search_longest_common_leading_whitespace() +in unicodeobject.c +*/ +static void +search_longest_common_leading_whitespace( + const char *const src, + const char *const end, + const char **indent, + Py_ssize_t *indent_len) +{ + // [_start, _start + _len) + // describes the current longest common leading whitespace + const char *_start = *indent; + Py_ssize_t _len = *indent_len; + + // skip the first line. for example: + // s = df""" + // first part + // first part{x}second part + // second part + // """ + // we don't need newline after opening qute. + // we don't need first line in the second part too. + const char *iter = memchr(src, '\n', end - src); + if (iter == NULL) { + // single line string + return; + } + + for (iter++; iter <= end; iter++) { + const char *line_start = iter; + const char *leading_whitespace_end = NULL; + + // scan the whole line + while (iter < end && *iter != '\n') { + if (!leading_whitespace_end && *iter != ' ' && *iter != '\t') { + /* `iter` points to the first non-whitespace character + in this line */ + if (iter == line_start) { + // some line has no indent, fast exit! + *indent = iter; + *indent_len = 0; + return; + } + leading_whitespace_end = iter; + } + ++iter; + } + + if (!leading_whitespace_end) { + // if this line has all white space, skip it + if (iter < end) { + continue; + } + leading_whitespace_end = iter; // last line may not end with '\n' + } + + if (!_start) { + // update the first leading whitespace + _start = line_start; + _len = leading_whitespace_end - line_start; + } + else { + /* We then compare with the current longest leading whitespace. + + [line_start, leading_whitespace_end) is the leading + whitespace of this line, + + [_start, _start + _len) is the leading whitespace of the + current longest leading whitespace. */ + Py_ssize_t new_len = 0; + const char *_iter = _start, *line_iter = line_start; + + while (_iter < _start + _len && line_iter < leading_whitespace_end + && *_iter == *line_iter) + { + ++_iter; + ++line_iter; + ++new_len; + } + + _len = new_len; + if (_len == 0) { + // No common things now, fast exit! + *indent = _start; + *indent_len = 0; + return; + } + } + } + + *indent = _start; + *indent_len = _len; +} + + static asdl_expr_seq * _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b, enum string_kind_t string_kind) { @@ -1343,12 +1562,53 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b return NULL; } int is_raw = strpbrk(quote_str, "rR") != NULL; + int is_dedent = strpbrk(quote_str, "dD") != NULL; asdl_expr_seq *seq = _Py_asdl_expr_seq_new(total_items, p->arena); if (seq == NULL) { return NULL; } + const char *indent_start = NULL; + Py_ssize_t indent_len = 0; + + if (is_dedent) { + if (total_items == 0) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, + "d-string must start with a newline" + ); + return NULL; + } + expr_ty first_item = asdl_seq_GET(raw_expressions, 0); + if (first_item->kind != Constant_kind + || PyUnicode_ReadChar(first_item->v.Constant.value, 0) != '\n') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + first_item, + "d-string must start with a newline" + ); + return NULL; + } + + for (Py_ssize_t i = 0; i < n_items; i++) { + expr_ty item = asdl_seq_GET(raw_expressions, i); + if (item->kind == Constant_kind) { + Py_ssize_t blen; + const char *bstr = PyUnicode_AsUTF8AndSize(item->v.Constant.value, &blen); + if (bstr == NULL) { + return NULL; + } + search_longest_common_leading_whitespace(bstr, bstr + blen, &indent_start, &indent_len); + } + } + + assert(indent_start != NULL); // TODO: is this assert true? + // _py_serach_longest_common_leading_whitespace() may not set indent_start when string is empty. + if (indent_len == 0) { + indent_start = ""; + } + } + Py_ssize_t index = 0; for (Py_ssize_t i = 0; i < n_items; i++) { expr_ty item = asdl_seq_GET(raw_expressions, i); @@ -1380,7 +1640,7 @@ _get_resized_exprs(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b } if (item->kind == Constant_kind) { - item = _PyPegen_decode_fstring_part(p, is_raw, item, b); + item = _PyPegen_decode_fstring_part(p, i == 0, is_raw, indent_start, indent_len, item, b); if (item == NULL) { return NULL; } @@ -1416,6 +1676,9 @@ expr_ty _PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b) { asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, TSTRING); + if (resized_exprs == NULL) { + return NULL; + } return _PyAST_TemplateStr(resized_exprs, a->lineno, a->col_offset, b->end_lineno, b->end_col_offset, p->arena); @@ -1425,6 +1688,9 @@ expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) { asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, FSTRING); + if (resized_exprs == NULL) { + return NULL; + } return _PyAST_JoinedStr(resized_exprs, a->lineno, a->col_offset, b->end_lineno, b->end_col_offset, p->arena); diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index ed1d2e08df8f400..e2a57177d20afb7 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -487,7 +487,7 @@ def visitField(self, sum): class Obj2ModPrototypeVisitor(PickleVisitor): def visitProduct(self, prod, name): - code = "static int obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena);" + code = "static int obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, const char* field, PyArena* arena);" self.emit(code % (name, get_c_type(name)), 0) visitSum = visitProduct @@ -511,7 +511,7 @@ def recursive_call(self, node, level): def funcHeader(self, name): ctype = get_c_type(name) self.emit("int", 0) - self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) + self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, const char* field, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) self.emit("int isinstance;", 1) self.emit("", 0) @@ -547,6 +547,18 @@ def simpleSum(self, sum, name): def buildArgs(self, fields): return ", ".join(fields + ["arena"]) + def typeCheck(self, name): + self.emit("tp = state->%s_type;" % name, 1) + self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1) + self.emit("if (isinstance == -1) {", 1) + self.emit("return 1;", 2) + self.emit("}", 1) + self.emit("if (!isinstance && field != NULL) {", 1) + error = "field '%%s' was expecting node of type '%s', got '%%s'" % name + self.emit("PyErr_Format(PyExc_TypeError, \"%s\", field, _PyType_Name(Py_TYPE(obj)));" % error, 2, reflow=False) + self.emit("return 1;", 2) + self.emit("}", 1) + def complexSum(self, sum, name): self.funcHeader(name) self.emit("PyObject *tmp = NULL;", 1) @@ -559,6 +571,7 @@ def complexSum(self, sum, name): self.emit("*out = NULL;", 2) self.emit("return 0;", 2) self.emit("}", 1) + self.typeCheck(name) for a in sum.attributes: self.visitField(a, name, sum=sum, depth=1) for t in sum.types: @@ -593,7 +606,7 @@ def visitSum(self, sum, name): def visitProduct(self, prod, name): ctype = get_c_type(name) self.emit("int", 0) - self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) + self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, const char* field, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) self.emit("PyObject* tmp = NULL;", 1) for f in prod.fields: @@ -694,8 +707,8 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): self.emit("%s val;" % ctype, depth+2) self.emit("PyObject *tmp2 = Py_NewRef(PyList_GET_ITEM(tmp, i));", depth+2) with self.recursive_call(name, depth+2): - self.emit("res = obj2ast_%s(state, tmp2, &val, arena);" % - field.type, depth+2, reflow=False) + self.emit("res = obj2ast_%s(state, tmp2, &val, \"%s\", arena);" % + (field.type, field.name), depth+2, reflow=False) self.emit("Py_DECREF(tmp2);", depth+2) self.emit("if (res != 0) goto failed;", depth+2) self.emit("if (len != PyList_GET_SIZE(tmp)) {", depth+2) @@ -709,8 +722,8 @@ def visitField(self, field, name, sum=None, prod=None, depth=0): self.emit("}", depth+1) else: with self.recursive_call(name, depth+1): - self.emit("res = obj2ast_%s(state, tmp, &%s, arena);" % - (field.type, field.name), depth+1) + self.emit("res = obj2ast_%s(state, tmp, &%s, \"%s\", arena);" % + (field.type, field.name, field.name), depth+1) self.emit("if (res != 0) goto failed;", depth+1) self.emit("Py_CLEAR(tmp);", depth+1) @@ -1701,7 +1714,9 @@ def visitModule(self, mod): /* Conversion Python -> AST */ -static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, + PyObject** out, + const char* Py_UNUSED(field), PyArena* arena) { if (obj == Py_None) obj = NULL; @@ -1718,7 +1733,9 @@ def visitModule(self, mod): return 0; } -static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, + PyObject** out, + const char* Py_UNUSED(field), PyArena* arena) { if (_PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; @@ -1728,29 +1745,29 @@ def visitModule(self, mod): return 0; } -static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, const char* field, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { - PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); + PyErr_Format(PyExc_TypeError, "field '%s' was expecting a string object", field); return -1; } - return obj2ast_object(state, obj, out, arena); + return obj2ast_object(state, obj, out, field, arena); } -static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, const char* field, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { - PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); + PyErr_Format(PyExc_TypeError, "field '%s' was expecting a string or bytes object", field); return -1; } - return obj2ast_object(state, obj, out, arena); + return obj2ast_object(state, obj, out, field, arena); } -static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) +static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, const char* field, PyArena* arena) { int i; if (!PyLong_Check(obj)) { - PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj); + PyErr_Format(PyExc_ValueError, "field \\"%s\\" got an invalid integer value: %R", field, obj); return -1; } @@ -2150,7 +2167,7 @@ class PartingShots(StaticVisitor): } mod_ty res = NULL; - if (obj2ast_mod(state, ast, &res, arena) != 0) + if (obj2ast_mod(state, ast, &res, NULL, arena) != 0) return NULL; else return res; diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 7f25afec302c225..07c61f1cb4386b3 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -455,7 +455,7 @@ tok_continuation_line(struct tok_state *tok) { static int maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, int saw_b, int saw_r, int saw_u, - int saw_f, int saw_t) { + int saw_f, int saw_t, int saw_d) { // Supported: rb, rf, rt (in any order) // Unsupported: ub, ur, uf, ut, bf, bt, ft (in any order) @@ -480,6 +480,9 @@ maybe_raise_syntax_error_for_string_prefixes(struct tok_state *tok, if (saw_u && saw_t) { RETURN_SYNTAX_ERROR("u", "t"); } + if (saw_u && saw_d) { + RETURN_SYNTAX_ERROR("u", "d"); + } if (saw_b && saw_f) { RETURN_SYNTAX_ERROR("b", "f"); @@ -741,8 +744,8 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t /* Identifier (most frequent token!) */ nonascii = 0; if (is_potential_identifier_start(c)) { - /* Process the various legal combinations of b"", r"", u"", and f"". */ - int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0; + /* Process the various legal combinations of b"", r"", u"", f"", and d"". */ + int saw_b = 0, saw_r = 0, saw_u = 0, saw_f = 0, saw_t = 0, saw_d = 0; while (1) { if (!saw_b && (c == 'b' || c == 'B')) { saw_b = 1; @@ -762,6 +765,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t else if (!saw_t && (c == 't' || c == 'T')) { saw_t = 1; } + else if (!saw_d && (c == 'd' || c == 'D')) { + saw_d = 1; + } else { break; } @@ -769,7 +775,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (c == '"' || c == '\'') { // Raise error on incompatible string prefixes: int status = maybe_raise_syntax_error_for_string_prefixes( - tok, saw_b, saw_r, saw_u, saw_f, saw_t); + tok, saw_b, saw_r, saw_u, saw_f, saw_t, saw_d); if (status < 0) { return MAKE_TOKEN(ERRORTOKEN); } @@ -1049,7 +1055,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } f_string_quote: - if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't') + if (((Py_TOLOWER(*tok->start) == 'f' || Py_TOLOWER(*tok->start) == 'r' || Py_TOLOWER(*tok->start) == 't' || Py_TOLOWER(*tok->start) == 'd') && (c == '\'' || c == '"'))) { int quote = c; @@ -1089,6 +1095,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->kind = TOK_FSTRING_MODE; the_current_tok->quote = quote; the_current_tok->quote_size = quote_size; + the_current_tok->raw = 0; the_current_tok->start = tok->start; the_current_tok->multi_line_start = tok->line_start; the_current_tok->first_line = tok->lineno; @@ -1101,25 +1108,28 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->in_debug = 0; enum string_kind_t string_kind = FSTRING; - switch (*tok->start) { - case 'T': - case 't': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - string_kind = TSTRING; - break; - case 'F': - case 'f': - the_current_tok->raw = Py_TOLOWER(*(tok->start + 1)) == 'r'; - break; - case 'R': - case 'r': - the_current_tok->raw = 1; - if (Py_TOLOWER(*(tok->start + 1)) == 't') { + for (const char *p = tok->start; *p != c; p++) { + switch (*p) { + case 'f': + case 'F': + break; + case 't': + case 'T': string_kind = TSTRING; - } - break; - default: - Py_UNREACHABLE(); + break; + case 'r': + case 'R': + the_current_tok->raw = 1; + break; + case 'd': + case 'D': + if (quote_size != 3) { + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "d-string must be triple-quoted")); + } + break; + default: + Py_UNREACHABLE(); + } } the_current_tok->string_kind = string_kind; diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 3663dc3eb7f9f69..5cf9b4d768c3ebb 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -15,8 +15,11 @@ _PyTokenizer_tok_new(void) struct tok_state *tok = (struct tok_state *)PyMem_Calloc( 1, sizeof(struct tok_state)); - if (tok == NULL) + if (tok == NULL) { + PyErr_NoMemory(); return NULL; + } + tok->buf = tok->cur = tok->inp = NULL; tok->fp_interactive = 0; tok->interactive_src_start = NULL; diff --git a/Parser/pegen.c b/Parser/pegen.c index 569f5afb3120085..bb222b50fc095f2 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -9,6 +9,7 @@ #include "lexer/lexer.h" #include "tokenizer/tokenizer.h" +#include "tokenizer/helpers.h" #include "pegen.h" // Internal parser functions @@ -993,7 +994,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2); if (tok == NULL) { if (PyErr_Occurred()) { - _PyPegen_raise_tokenizer_init_error(filename_ob); + _PyTokenizer_raise_init_error(filename_ob); return NULL; } return NULL; @@ -1051,7 +1052,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } if (tok == NULL) { if (PyErr_Occurred()) { - _PyPegen_raise_tokenizer_init_error(filename_ob); + _PyTokenizer_raise_init_error(filename_ob); } return NULL; } diff --git a/Parser/pegen.h b/Parser/pegen.h index 85c9ada765d9bd4..5c461e82a7f0fa7 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -174,7 +174,6 @@ typedef enum { } TARGETS_TYPE; int _Pypegen_raise_decode_error(Parser *p); -void _PyPegen_raise_tokenizer_init_error(PyObject *filename); int _Pypegen_tokenizer_error(Parser *p); void *_PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...); void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 312699415efd9af..b13e1c079220a92 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -10,53 +10,6 @@ // TOKENIZER ERRORS -void -_PyPegen_raise_tokenizer_init_error(PyObject *filename) -{ - if (!(PyErr_ExceptionMatches(PyExc_LookupError) - || PyErr_ExceptionMatches(PyExc_SyntaxError) - || PyErr_ExceptionMatches(PyExc_ValueError) - || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) { - return; - } - PyObject *errstr = NULL; - PyObject *tuple = NULL; - PyObject *type; - PyObject *value; - PyObject *tback; - PyErr_Fetch(&type, &value, &tback); - if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) { - if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) { - goto error; - } - PyErr_Restore(type, value, tback); - return; - } - errstr = PyObject_Str(value); - if (!errstr) { - goto error; - } - - PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None); - if (!tmp) { - goto error; - } - - tuple = _PyTuple_FromPair(errstr, tmp); - Py_DECREF(tmp); - if (!tuple) { - goto error; - } - PyErr_SetObject(PyExc_SyntaxError, tuple); - -error: - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(tback); - Py_XDECREF(errstr); - Py_XDECREF(tuple); -} - static inline void raise_unclosed_parentheses_error(Parser *p) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index b164dfbc81a9339..3425b856796fe5a 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -247,6 +247,67 @@ _PyPegen_decode_string(Parser *p, int raw, const char *s, size_t len, Token *t) return decode_unicode_with_escapes(p, s, len, t); } +/* defined in unicodeobject.c */ +extern Py_ssize_t +_Py_search_longest_common_leading_whitespace( + const char *const src, + const char *const end, + const char **output + ); + +// Dedent d-string and return result as a bytes. +static PyObject* +_PyPegen_dedent_string(Parser *p, const char *s, Py_ssize_t len, + const char *indent, Py_ssize_t indent_len, int lineno) +{ + PyBytesWriter *w = PyBytesWriter_Create(0); + if (w == NULL) { + return NULL; + } + + const char *end = s + len; + for (; s < end; lineno++) { + Py_ssize_t i; + for (i = 0; i < indent_len; i++) { + if (s[i] != indent[i]) { + if (s[i] == '\n') { + break; // empty line + } + PyBytesWriter_Discard(w); + RAISE_ERROR_KNOWN_LOCATION(p, PyExc_IndentationError, lineno, i, lineno, i+1, + "d-string missing valid indentation"); + return NULL; + } + } + + if (s[i] == '\n') { // found an empty line with newline. + if (PyBytesWriter_WriteBytes(w, "\n", 1) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } + s += i+1; + continue; + } + + // found a indented line. let's dedent it. + s += i; + const char *line_end = memchr(s, '\n', end - s); + if (line_end == NULL) { + line_end = end; // last line without newline + } + else { + line_end++; // include the newline in the line + } + + if (PyBytesWriter_WriteBytes(w, s, line_end - s) < 0) { + PyBytesWriter_Discard(w); + return NULL; + } + s = line_end; + } + return PyBytesWriter_Finish(w); +} + /* s must include the bracketing quote characters, and r, b &/or f prefixes (if any), and embedded escape sequences (if any). (f-strings are handled by the parser) _PyPegen_parse_string parses it, and returns the decoded Python string object. */ @@ -262,9 +323,10 @@ _PyPegen_parse_string(Parser *p, Token *t) int quote = Py_CHARMASK(*s); int bytesmode = 0; int rawmode = 0; + int dedentmode = 0; if (Py_ISALPHA(quote)) { - while (!bytesmode || !rawmode) { + while (!bytesmode || !rawmode || !dedentmode) { if (quote == 'b' || quote == 'B') { quote =(unsigned char)*++s; bytesmode = 1; @@ -276,6 +338,10 @@ _PyPegen_parse_string(Parser *p, Token *t) quote = (unsigned char)*++s; rawmode = 1; } + else if (quote == 'd' || quote == 'D') { + quote =(unsigned char)*++s; + dedentmode = 1; + } else { break; } @@ -315,9 +381,64 @@ _PyPegen_parse_string(Parser *p, Token *t) return NULL; } } + else if (dedentmode) { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "d-string must be triple-quoted"); + return NULL; + } /* Avoid invoking escape decoding routines if possible. */ rawmode = rawmode || strchr(s, '\\') == NULL; + + int _prev_call_invald = p->call_invalid_rules; + + PyObject *dedent_bytes = NULL; + if (dedentmode) { + if (len == 0 || s[0] != '\n') { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(t, "d-string must start with a newline"); + return NULL; + } + + // _PyPegen_decode_string() and decode_bytes_with_escapes() emit + // a warning for invalid escape sequences. + // We need to call it before dedenting since it shifts the positions. + if (!_prev_call_invald && !rawmode) { + PyObject *temp; + if (bytesmode) { + temp = decode_bytes_with_escapes(p, s, len, t); + } + else { + temp = _PyPegen_decode_string(p, 0, s, len, t); + } + if (temp == NULL) { + return NULL; + } + Py_DECREF(temp); + } + + // We find common indent from [s, end+1) because we want to include the last line + // for indent calculation. + const char *end = s + len; + assert(*end == '"' || *end == '\''); // end[0:3] is the trailing quotes + const char *indent; + Py_ssize_t indent_len = _Py_search_longest_common_leading_whitespace(s+1, end+1, &indent); + + s++; len--; // skip the first newline + if (indent_len > 0) { + // dedent the string + dedent_bytes = _PyPegen_dedent_string(p, s, len, indent, indent_len, t->lineno + 1); + if (dedent_bytes == NULL) { + return NULL; + } + if (PyBytes_AsStringAndSize(dedent_bytes, (char**)&s, (Py_ssize_t*)&len) < 0) { + Py_DECREF(dedent_bytes); + return NULL; + } + } + + p->call_invalid_rules = 1; + } + + PyObject *result; if (bytesmode) { /* Disallow non-ASCII characters. */ const char *ch; @@ -327,13 +448,23 @@ _PyPegen_parse_string(Parser *p, Token *t) t, "bytes can only contain ASCII " "literal characters"); + Py_XDECREF(dedent_bytes); + p->call_invalid_rules = _prev_call_invald; return NULL; } } if (rawmode) { - return PyBytes_FromStringAndSize(s, (Py_ssize_t)len); + result = PyBytes_FromStringAndSize(s, (Py_ssize_t)len); } - return decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); + else { + result = decode_bytes_with_escapes(p, s, (Py_ssize_t)len, t); + } + } + else { + result = _PyPegen_decode_string(p, rawmode, s, len, t); } - return _PyPegen_decode_string(p, rawmode, s, len, t); + Py_XDECREF(dedent_bytes); + p->call_invalid_rules = _prev_call_invald; + return result; } + diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index 8c836a3f7258296..a11702557a07af3 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -378,6 +378,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 9542969ad3127b9..62b0971d418c396 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -1,6 +1,8 @@ #include "Python.h" #include "errcode.h" +#include "pycore_runtime.h" // _Py_ID() #include "pycore_token.h" +#include "pycore_tuple.h" // _PyTuple_FromPair #include "../lexer/state.h" @@ -149,6 +151,53 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval return 0; } +void +_PyTokenizer_raise_init_error(PyObject *filename) +{ + if (!(PyErr_ExceptionMatches(PyExc_LookupError) + || PyErr_ExceptionMatches(PyExc_SyntaxError) + || PyErr_ExceptionMatches(PyExc_ValueError) + || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) { + return; + } + PyObject *errstr = NULL; + PyObject *tuple = NULL; + PyObject *type; + PyObject *value; + PyObject *tback; + PyErr_Fetch(&type, &value, &tback); + if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) { + if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) { + goto error; + } + PyErr_Restore(type, value, tback); + return; + } + errstr = PyObject_Str(value); + if (!errstr) { + goto error; + } + + PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None); + if (!tmp) { + goto error; + } + + tuple = _PyTuple_FromPair(errstr, tmp); + Py_DECREF(tmp); + if (!tuple) { + goto error; + } + PyErr_SetObject(PyExc_SyntaxError, tuple); + +error: + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(tback); + Py_XDECREF(errstr); + Py_XDECREF(tuple); +} + int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { @@ -193,6 +242,7 @@ _PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok) char* result = (char *)PyMem_Malloc(len + 1); if (!result) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } memcpy(result, s, len); @@ -221,6 +271,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf buf = PyMem_Malloc(needed_length); if (buf == NULL) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } for (current = buf; *s; s++, current++) { @@ -416,8 +467,8 @@ _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_sta if (tok->encoding == NULL) { assert(tok->decoding_readline == NULL); if (strcmp(cs, "utf-8") != 0 && !set_readline(tok, cs)) { + _PyTokenizer_raise_init_error(tok->filename); _PyTokenizer_error_ret(tok); - PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs); PyMem_Free(cs); return 0; } diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 98f6445d5a3b40e..34303999a60aff7 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -15,6 +15,7 @@ int _PyTokenizer_indenterror(struct tok_state *tok); int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); char *_PyTokenizer_error_ret(struct tok_state *tok); +void _PyTokenizer_raise_init_error(PyObject *filename); char *_PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok); char *_PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok); diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index 0f7769aeb8fd570..917f7b40cfbbfed 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -114,6 +114,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Platforms/Android/__main__.py b/Platforms/Android/__main__.py index d2546cf76c206b0..063e1f316031443 100755 --- a/Platforms/Android/__main__.py +++ b/Platforms/Android/__main__.py @@ -219,8 +219,8 @@ def unpack_deps(host, prefix_dir, cache_dir): for name_ver in [ "bzip2-1.0.8-3", "libffi-3.4.4-3", - "openssl-3.5.6-0", - "sqlite-3.50.4-0", + "openssl-3.5.7-0", + "sqlite-3.53.2-0", "xz-5.4.6-1", "zstd-1.5.7-2" ]: diff --git a/Platforms/Android/testbed/app/build.gradle.kts b/Platforms/Android/testbed/app/build.gradle.kts index e51398fce81e268..a56714406803cbf 100644 --- a/Platforms/Android/testbed/app/build.gradle.kts +++ b/Platforms/Android/testbed/app/build.gradle.kts @@ -105,12 +105,13 @@ android { // This controls the API level of the maxVersion managed emulator, which is used // by CI and cibuildwheel. + // * 32 has intermittent failures accessing the internet (#142387). // * 33 has excessive buffering in the logcat client // (https://cs.android.com/android/_/android/platform/system/logging/+/d340721894f223327339010df59b0ac514308826). - // * 34 consumes too much disk space on GitHub Actions (#142289). - // * 35 has issues connecting to the internet (#142387). + // * 34 consumes too much disk space on GitHub Actions (#142289), though switching to the + // "default" image may be a workaround. // * 36 and later are not available as aosp_atd images yet. - targetSdk = 32 + targetSdk = 35 versionCode = 1 versionName = "1.0" diff --git a/Platforms/Apple/__main__.py b/Platforms/Apple/__main__.py index d94198a309f9269..9f2d2afb0aa0f67 100644 --- a/Platforms/Apple/__main__.py +++ b/Platforms/Apple/__main__.py @@ -319,7 +319,7 @@ def unpack_deps( for name_ver in [ "BZip2-1.0.8-2", "libFFI-3.4.7-2", - "OpenSSL-3.5.6-1", + "OpenSSL-3.5.7-1", "XZ-5.6.4-2", "mpdecimal-4.0.0-2", "zstd-1.5.7-1", diff --git a/Platforms/Apple/testbed/__main__.py b/Platforms/Apple/testbed/__main__.py index 0dd77ab8b827974..b3eed38571d9708 100644 --- a/Platforms/Apple/testbed/__main__.py +++ b/Platforms/Apple/testbed/__main__.py @@ -21,7 +21,7 @@ LOG_PREFIX_REGEX = re.compile( r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}" # HH:MM:SS.ssssss+ZZZZ - r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID + r"\s+iOSTestbed\[\d+:\w+\] " # Process/thread ID ) diff --git a/Platforms/WASI/__main__.py b/Platforms/WASI/__main__.py index b8513a004f18e5f..880058bad660be9 100644 --- a/Platforms/WASI/__main__.py +++ b/Platforms/WASI/__main__.py @@ -40,6 +40,9 @@ def main(): build_python = subcommands.add_parser( "build-python", help="Build the build Python" ) + pythoninfo_build = subcommands.add_parser( + "pythoninfo-build", help="Display build info of the build Python" + ) configure_host = subcommands.add_parser( "configure-host", help="Run `configure` for the " @@ -53,6 +56,9 @@ def main(): build_host = subcommands.add_parser( "build-host", help="Build the host/WASI Python" ) + pythoninfo_host = subcommands.add_parser( + "pythoninfo-host", help="Display build info of the host/WASI Python" + ) subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) @@ -61,8 +67,10 @@ def main(): configure_build, make_build, build_python, + pythoninfo_build, configure_host, make_host, + pythoninfo_host, build_host, ): subcommand.add_argument( @@ -118,7 +126,13 @@ def main(): help="Command template for running the WASI host; defaults to " f"`{default_host_runner}`", ) - for subcommand in build, configure_host, make_host, build_host: + for subcommand in ( + build, + configure_host, + make_host, + build_host, + pythoninfo_host, + ): subcommand.add_argument( "--host-triple", action="store", @@ -137,6 +151,8 @@ def main(): case "build-python": _build.configure_build_python(context) _build.make_build_python(context) + case "pythoninfo-build": + _build.pythoninfo_build_python(context) case "configure-host": _build.configure_wasi_python(context) case "make-host": @@ -144,13 +160,22 @@ def main(): case "build-host": _build.configure_wasi_python(context) _build.make_wasi_python(context) + case "pythoninfo-host": + _build.pythoninfo_wasi_python(context) case "build": + # Configure and build the build Python _build.configure_build_python(context) _build.make_build_python(context) + _build.pythoninfo_build_python(context) + + # Configure and build the host/WASI Python _build.configure_wasi_python(context) _build.make_wasi_python(context) + _build.pythoninfo_wasi_python(context) case "clean": _build.clean_contents(context) + case None: + parser.print_help() case _: raise ValueError(f"Unknown subcommand {context.subcommand!r}") diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 76d2853163baa9e..792f8edd9943138 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -222,10 +222,8 @@ def wasi_sdk(context): if wasi_sdk_path := context.wasi_sdk_path: if not wasi_sdk_path.exists(): raise ValueError( - "WASI SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk" + "WASI SDK not found at " + f"{os.fsdecode(wasi_sdk_path)!r} (via --wasi-sdk)" ) return wasi_sdk_path @@ -237,7 +235,8 @@ def wasi_sdk(context): wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) if not wasi_sdk_path.exists(): raise ValueError( - f"WASI SDK not found at $WASI_SDK_PATH ({wasi_sdk_path})" + f"WASI SDK not found at {os.fsdecode(wasi_sdk_path)!r} " + "(via $WASI_SDK_PATH)" ) else: opt_path = pathlib.Path("/opt") @@ -272,6 +271,14 @@ def wasi_sdk(context): f" Found WASI SDK {major_version}, " f"but WASI SDK {wasi_sdk_version} is the supported version", ) + elif not wasi_sdk_path: + raise ValueError( + f"WASI SDK {wasi_sdk_version} not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and install in " + f"{os.fsdecode(opt_path)!r} or specify the SDK via " + "$WASI_SDK_PATH or --wasi-sdk" + ) # Cache the result. context.wasi_sdk_path = wasi_sdk_path @@ -411,3 +418,15 @@ def clean_contents(context): if LOCAL_SETUP.exists(): if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: log("🧹", f"Deleting generated {LOCAL_SETUP} ...") + + +@subdir(BUILD_DIR) +def pythoninfo_build_python(context, working_dir): + """Display build info of the build Python.""" + call(["make", "pythoninfo"], context=context) + + +@subdir(lambda context: CROSS_BUILD_DIR / host_triple(context)) +def pythoninfo_wasi_python(context, working_dir): + """Display build info of the host/WASI Python.""" + call(["make", "pythoninfo"], context=context) diff --git a/Platforms/emscripten/__main__.py b/Platforms/emscripten/__main__.py index c1eac8005474fda..84a1589b81f6a20 100644 --- a/Platforms/emscripten/__main__.py +++ b/Platforms/emscripten/__main__.py @@ -290,6 +290,12 @@ def make_build_python(context, working_dir): print(f"🎉 {binary} {version}") +@subdir("native_build_dir") +def pythoninfo_build_python(context, working_dir): + """Display build info of the build Python.""" + call(["make", "pythoninfo"], quiet=context.quiet) + + def check_shasum(file: str, expected_shasum: str): with open(file, "rb") as f: digest = hashlib.file_digest(f, "sha256") @@ -580,7 +586,7 @@ def make_emscripten_python(context, working_dir): subprocess.check_call([exec_script, "--version"]) -def run_emscripten_python(context): +def run_emscripten_python(context, args=None): """Run the built emscripten Python.""" host_dir = context.build_paths["host_dir"] exec_script = host_dir / "python.sh" @@ -588,19 +594,25 @@ def run_emscripten_python(context): print("Emscripten not built", file=sys.stderr) sys.exit(1) - args = context.args - # Strip the "--" separator if present - if args and args[0] == "--": - args = args[1:] + if args is None: + args = context.args + # Strip the "--" separator if present + if args and args[0] == "--": + args = args[1:] - if context.test: - args = load_config_toml()["test-args"] + args - elif context.pythoninfo: - args = load_config_toml()["pythoninfo-args"] + args + if context.test: + args = load_config_toml()["test-args"] + args + elif context.pythoninfo: + args = load_config_toml()["pythoninfo-args"] + args os.execv(str(exec_script), [str(exec_script), *args]) +def pythoninfo_emscripten_python(context): + """Display build info of the host/Emscripten Python.""" + run_emscripten_python(context, ["-m", "test.pythoninfo"]) + + def build_target(context): """Build one or more targets.""" steps = [] @@ -608,6 +620,7 @@ def build_target(context): steps.extend([ configure_build_python, make_build_python, + pythoninfo_build_python, ]) if context.target in {"host", "all"}: steps.extend([ @@ -615,6 +628,7 @@ def build_target(context): make_mpdec, configure_emscripten_python, make_emscripten_python, + pythoninfo_emscripten_python, ]) for step in steps: @@ -650,7 +664,7 @@ def add_cross_build_dir_option(subcommand): help=( "Path to the cross-build directory " f"(default: {DEFAULT_CROSS_BUILD_DIR}). " - "Can also be set with the CROSS_BUILD_DIR environment variable.", + "Can also be set with the CROSS_BUILD_DIR environment variable." ), ) @@ -707,6 +721,10 @@ def main(): "make-build-python", help="Run `make` for the build Python" ) + pythoninfo_build = subcommands.add_parser( + "pythoninfo-build", help="Display build info of the build Python" + ) + configure_host = subcommands.add_parser( "configure-host", help=( @@ -719,6 +737,10 @@ def main(): "make-host", help="Run `make` for the host/emscripten" ) + pythoninfo_host = subcommands.add_parser( + "pythoninfo-host", help="Display build info of the host/Emscripten Python" + ) + run = subcommands.add_parser( "run", help="Run the built emscripten Python", @@ -743,7 +765,7 @@ def main(): nargs=argparse.REMAINDER, help=( "Arguments to pass to the emscripten Python " - "(use '--' to separate from run options)", + "(use '--' to separate from run options)" ), ) add_cross_build_dir_option(run) @@ -770,8 +792,10 @@ def main(): make_mpdec_cmd, make_dependencies_cmd, make_build, + pythoninfo_build, configure_host, make_host, + pythoninfo_host, clean, ): subcommand.add_argument( @@ -840,8 +864,10 @@ def main(): "make-dependencies": make_dependencies, "configure-build-python": configure_build_python, "make-build-python": make_build_python, + "pythoninfo-build": pythoninfo_build_python, "configure-host": configure_emscripten_python, "make-host": make_emscripten_python, + "pythoninfo-host": pythoninfo_emscripten_python, "build": build_target, "run": run_emscripten_python, "clean": clean_contents, diff --git a/Platforms/emscripten/config.toml b/Platforms/emscripten/config.toml index ba2dc8f4a482bfa..389d2ea66ce948e 100644 --- a/Platforms/emscripten/config.toml +++ b/Platforms/emscripten/config.toml @@ -1,7 +1,7 @@ # Any data that can vary between Python versions is to be kept in this file. # This allows for blanket copying of the Emscripten build code between supported # Python versions. -emscripten-version = "4.0.12" +emscripten-version = "6.0.0" node-version = "24" test-args = [ "-m", "test", diff --git a/Platforms/emscripten/node_entry.mjs b/Platforms/emscripten/node_entry.mjs index 9478b7714adbc80..110aadc5de10143 100644 --- a/Platforms/emscripten/node_entry.mjs +++ b/Platforms/emscripten/node_entry.mjs @@ -57,6 +57,6 @@ try { // Show JavaScript exception and traceback console.warn(e); // Show Python exception and traceback - Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + Module.PyUnstable_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); process.exit(1); } diff --git a/Platforms/emscripten/streams.mjs b/Platforms/emscripten/streams.mjs index 76ad79f9247f4cf..1b121d48d4e76c6 100644 --- a/Platforms/emscripten/streams.mjs +++ b/Platforms/emscripten/streams.mjs @@ -112,7 +112,7 @@ const prepareBuffer = (buffer, offset, length) => const TTY_OPS = { ioctl_tiocgwinsz(tty) { - return tty.devops.ioctl_tiocgwinsz?.(); + return tty.devops.ioctl_tiocgwinsz?.() ?? [24, 80]; }, }; @@ -188,6 +188,10 @@ class NodeReader { fsync() { nodeFsync(this.nodeStream.fd); } + + ioctl_tiocgwinsz() { + return [this.nodeStream.rows ?? 24, this.nodeStream.columns ?? 80]; + } } class NodeWriter { diff --git a/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs b/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs index 5642372c9d24728..38a622117c2a50f 100644 --- a/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs +++ b/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs @@ -189,6 +189,6 @@ try { // Show JavaScript exception and traceback console.warn(e); // Show Python exception and traceback - Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + Module.PyUnstable_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); process.exit(1); } diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 285f4f091b2f7ab..74d0fe37ddaeadb 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -10,6 +10,7 @@ #include "pycore_runtime.h" // _PyRuntime #include "pycore_lock.h" // PyEvent #include "pycore_pythread.h" // PyThread_start_joinable_thread() +#include "pycore_pystate.h" // _PyInterpreterState_GuardCountdown #include "pycore_import.h" // _PyImport_FrozenBootstrap #include <inttypes.h> #include <stdio.h> @@ -40,8 +41,13 @@ char **main_argv; #define PROGRAM "test_embed" /* Use path starting with "./" avoids a search along the PATH */ -#define PROGRAM_NAME L"./_testembed" -#define PROGRAM_NAME_UTF8 "./_testembed" +#ifdef __CYGWIN__ +# define PROGRAM_NAME L"./_testembed.exe" +# define PROGRAM_NAME_UTF8 "./_testembed.exe" +#else +# define PROGRAM_NAME L"./_testembed" +# define PROGRAM_NAME_UTF8 "./_testembed" +#endif #define INIT_LOOPS 4 @@ -2670,6 +2676,214 @@ test_gilstate_after_finalization(void) return PyThread_detach_thread(handle); } + +const char *THREAD_CODE = \ + "import time\n" + "time.sleep(0.2)\n" + "def fib(n):\n" + " if n <= 1:\n" + " return n\n" + " else:\n" + " return fib(n - 1) + fib(n - 2)\n" + "fib(10)"; + +typedef struct { + void *argument; + int done; + PyEvent event; +} ThreadData; + +static void +do_tstate_ensure(void *arg) +{ + ThreadData *data = (ThreadData *)arg; + PyThreadStateToken *tokens[4]; + PyInterpreterGuard *guard = data->argument; + tokens[0] = PyThreadState_Ensure(guard); + tokens[1] = PyThreadState_Ensure(guard); + tokens[2] = PyThreadState_Ensure(guard); + PyGILState_STATE gstate = PyGILState_Ensure(); + tokens[3] = PyThreadState_Ensure(guard); + assert(tokens[0] != NULL); + assert(tokens[1] != NULL); + assert(tokens[2] != NULL); + assert(tokens[3] != NULL); + int res = PyRun_SimpleString(THREAD_CODE); + assert(res == 0); + PyThreadState_Release(tokens[3]); + PyGILState_Release(gstate); + PyThreadState_Release(tokens[2]); + PyThreadState_Release(tokens[1]); + PyThreadState_Release(tokens[0]); + _Py_atomic_store_int(&data->done, 1); + PyInterpreterGuard_Close(guard); +} + +static int +test_thread_state_ensure(void) +{ + _testembed_initialize(); + assert(_PyInterpreterState_GuardCountdown(_PyInterpreterState_GET()) == 0); + PyThread_handle_t handle; + PyThread_ident_t ident; + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + assert(guard != NULL); + ThreadData data = { guard }; + if (PyThread_start_joinable_thread(do_tstate_ensure, &data, + &ident, &handle) < 0) { + PyInterpreterGuard_Close(guard); + return -1; + } + // We hold an interpreter guard, so we don't + // have to worry about the interpreter shutting down before + // we finalize. + Py_Finalize(); + assert(_Py_atomic_load_int(&data.done) == 1); + PyThread_join_thread(handle); + return 0; +} + +static int +test_main_interpreter_view(void) +{ + PyInterpreterView *view = PyInterpreterView_FromMain(); + assert(view != NULL); + // These should fail -- the main interpreter is not available yet. + assert(PyInterpreterGuard_FromView(view) == NULL); + assert(PyThreadState_EnsureFromView(view) == NULL); + + _testembed_initialize(); + assert(_PyInterpreterState_GuardCountdown(_PyInterpreterState_GET()) == 0); + // Main interpreter is initialized and ready at this point. + + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + assert(guard != NULL); + PyInterpreterGuard_Close(guard); + + Py_Finalize(); + + // We shouldn't be able to get locks for the interpreter now + guard = PyInterpreterGuard_FromView(view); + assert(guard == NULL); + + PyInterpreterView_Close(view); + + return 0; +} + +static void +do_tstate_ensure_from_view(void *arg) +{ + ThreadData *data = (ThreadData *)arg; + PyInterpreterView *view = data->argument; + assert(view != NULL); + PyThreadStateToken *token = PyThreadState_EnsureFromView(view); + assert(token != NULL); + _PyEvent_Notify(&data->event); + int res = PyRun_SimpleString(THREAD_CODE); + assert(res == 0); + _Py_atomic_store_int(&data->done, 1); + PyThreadState_Release(token); +} + +static int +test_thread_state_ensure_from_view(void) +{ + _testembed_initialize(); + assert(_PyInterpreterState_GuardCountdown(_PyInterpreterState_GET()) == 0); + PyThread_handle_t handle; + PyThread_ident_t ident; + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + assert(view != NULL); + + ThreadData data = { view }; + if (PyThread_start_joinable_thread(do_tstate_ensure_from_view, &data, + &ident, &handle) < 0) { + PyInterpreterView_Close(view); + return -1; + } + + PyEvent_Wait(&data.event); + Py_Finalize(); + assert(_Py_atomic_load_int(&data.done) == 1); + PyThread_join_thread(handle); + return 0; +} + +#define NUM_THREADS 4 + +static void +stress_func(void *arg) +{ + PyInterpreterGuard *guard = (PyInterpreterGuard *)arg; + + for (int i = 0; i < 1000; ++i) { + assert(guard != NULL); + PyThreadStateToken *token = PyThreadState_Ensure(guard); + assert(token != NULL); + + PyGILState_STATE gstate = PyGILState_Ensure(); + + PyInterpreterView *view = PyInterpreterView_FromCurrent(); + assert(view != NULL); + + PyThreadStateToken *token2 = PyThreadState_EnsureFromView(view); + assert(token2 != NULL); + PyThreadState_Release(token2); + + PyGILState_Release(gstate); + + PyThreadState_Release(token); + + PyInterpreterGuard_Close(guard); + + guard = PyInterpreterGuard_FromView(view); + PyInterpreterView_Close(view); + + if (guard == NULL) { + // The interpreter is shutting down. Bail out now. + return; + } + } + + PyInterpreterGuard_Close(guard); +} + +static int +test_concurrent_finalization_stress(void) +{ + for (int j = 0; j < 50; ++j) { + _testembed_initialize(); + assert(_PyInterpreterState_GuardCountdown(_PyInterpreterState_GET()) == 0); + PyThread_handle_t handles[NUM_THREADS]; + PyThread_ident_t idents[NUM_THREADS]; + PyInterpreterGuard *guards[NUM_THREADS]; + + for (int i = 0; i < NUM_THREADS; ++i) { + guards[i] = PyInterpreterGuard_FromCurrent(); + assert(guards[i] != NULL); + if (PyThread_start_joinable_thread(stress_func, guards[i], &idents[i], &handles[i]) < 0) { + for (int x = 0; x < i; ++x) { + PyInterpreterGuard_Close(guards[x]); + PyThread_detach_thread(handles[x]); + } + return -1; + } + } + + Py_Finalize(); + + for (int i = 0; i < NUM_THREADS; ++i) { + PyThread_join_thread(handles[i]); + } + } + + return 0; +} + +#undef NUM_THREADS + + /* ********************************************************* * List of test cases and the function that implements it. * @@ -2764,6 +2978,10 @@ static struct TestCase TestCases[] = { {"test_create_module_from_initfunc", test_create_module_from_initfunc}, {"test_inittab_submodule_multiphase", test_inittab_submodule_multiphase}, {"test_inittab_submodule_singlephase", test_inittab_submodule_singlephase}, + {"test_thread_state_ensure", test_thread_state_ensure}, + {"test_main_interpreter_view", test_main_interpreter_view}, + {"test_thread_state_ensure_from_view", test_thread_state_ensure_from_view}, + {"test_concurrent_finalization_stress", test_concurrent_finalization_stress}, {NULL, NULL} }; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 5f3d9c4b17410fb..49b6bf1d12b6fab 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -6026,7 +6026,9 @@ static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) /* Conversion Python -> AST */ -static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, + PyObject** out, + const char* Py_UNUSED(field), PyArena* arena) { if (obj == Py_None) obj = NULL; @@ -6043,7 +6045,9 @@ static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyO return 0; } -static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, + PyObject** out, + const char* Py_UNUSED(field), PyArena* arena) { if (_PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; @@ -6053,29 +6057,29 @@ static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, P return 0; } -static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, const char* field, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { - PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); + PyErr_Format(PyExc_TypeError, "field '%s' was expecting a string object", field); return -1; } - return obj2ast_object(state, obj, out, arena); + return obj2ast_object(state, obj, out, field, arena); } -static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, const char* field, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { - PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); + PyErr_Format(PyExc_TypeError, "field '%s' was expecting a string or bytes object", field); return -1; } - return obj2ast_object(state, obj, out, arena); + return obj2ast_object(state, obj, out, field, arena); } -static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) +static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, const char* field, PyArena* arena) { int i; if (!PyLong_Check(obj)) { - PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj); + PyErr_Format(PyExc_ValueError, "field \"%s\" got an invalid integer value: %R", field, obj); return -1; } @@ -6954,43 +6958,52 @@ init_types(void *arg) } static int obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, - PyArena* arena); + const char* field, PyArena* arena); static int obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, - PyArena* arena); + const char* field, PyArena* arena); static int obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, - PyArena* arena); + const char* field, PyArena* arena); static int obj2ast_expr_context(struct ast_state *state, PyObject* obj, - expr_context_ty* out, PyArena* arena); + expr_context_ty* out, const char* field, + PyArena* arena); static int obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* - out, PyArena* arena); + out, const char* field, PyArena* arena); static int obj2ast_operator(struct ast_state *state, PyObject* obj, - operator_ty* out, PyArena* arena); + operator_ty* out, const char* field, PyArena* + arena); static int obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* - out, PyArena* arena); + out, const char* field, PyArena* arena); static int obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, - PyArena* arena); + const char* field, PyArena* arena); static int obj2ast_comprehension(struct ast_state *state, PyObject* obj, - comprehension_ty* out, PyArena* arena); + comprehension_ty* out, const char* field, + PyArena* arena); static int obj2ast_excepthandler(struct ast_state *state, PyObject* obj, - excepthandler_ty* out, PyArena* arena); + excepthandler_ty* out, const char* field, + PyArena* arena); static int obj2ast_arguments(struct ast_state *state, PyObject* obj, - arguments_ty* out, PyArena* arena); + arguments_ty* out, const char* field, PyArena* + arena); static int obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, - PyArena* arena); + const char* field, PyArena* arena); static int obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* - out, PyArena* arena); + out, const char* field, PyArena* arena); static int obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, - PyArena* arena); + const char* field, PyArena* arena); static int obj2ast_withitem(struct ast_state *state, PyObject* obj, - withitem_ty* out, PyArena* arena); + withitem_ty* out, const char* field, PyArena* + arena); static int obj2ast_match_case(struct ast_state *state, PyObject* obj, - match_case_ty* out, PyArena* arena); + match_case_ty* out, const char* field, PyArena* + arena); static int obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* - out, PyArena* arena); + out, const char* field, PyArena* arena); static int obj2ast_type_ignore(struct ast_state *state, PyObject* obj, - type_ignore_ty* out, PyArena* arena); + type_ignore_ty* out, const char* field, PyArena* + arena); static int obj2ast_type_param(struct ast_state *state, PyObject* obj, - type_param_ty* out, PyArena* arena); + type_param_ty* out, const char* field, PyArena* + arena); mod_ty _PyAST_Module(asdl_stmt_seq * body, asdl_type_ignore_seq * type_ignores, @@ -10855,7 +10868,8 @@ ast2obj_type_param(struct ast_state *state, void* _o) int -obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) +obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, const char* + field, PyArena* arena) { int isinstance; @@ -10866,6 +10880,15 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) *out = NULL; return 0; } + tp = state->mod_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (!isinstance && field != NULL) { + PyErr_Format(PyExc_TypeError, "field '%s' was expecting node of type 'mod', got '%s'", field, _PyType_Name(Py_TYPE(obj))); + return 1; + } tp = state->Module_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { @@ -10901,7 +10924,7 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'Module' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -10939,7 +10962,7 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'Module' node")) { goto failed; } - res = obj2ast_type_ignore(state, tmp2, &val, arena); + res = obj2ast_type_ignore(state, tmp2, &val, "type_ignores", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -10989,7 +11012,7 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'Interactive' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11025,7 +11048,7 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'Expression' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &body, arena); + res = obj2ast_expr(state, tmp, &body, "body", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11069,7 +11092,7 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'FunctionType' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "argtypes", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11093,7 +11116,7 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'FunctionType' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &returns, arena); + res = obj2ast_expr(state, tmp, &returns, "returns", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11110,8 +11133,8 @@ obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) } int -obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* - arena) +obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, const char* + field, PyArena* arena) { int isinstance; @@ -11126,6 +11149,15 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* *out = NULL; return 0; } + tp = state->stmt_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (!isinstance && field != NULL) { + PyErr_Format(PyExc_TypeError, "field '%s' was expecting node of type 'stmt', got '%s'", field, _PyType_Name(Py_TYPE(obj))); + return 1; + } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { return -1; } @@ -11138,7 +11170,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'stmt' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11155,7 +11187,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'stmt' node")) { goto failed; } - res = obj2ast_int(state, tmp, &col_offset, arena); + res = obj2ast_int(state, tmp, &col_offset, "col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11172,7 +11204,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'stmt' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_lineno, arena); + res = obj2ast_int(state, tmp, &end_lineno, "end_lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11189,7 +11221,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'stmt' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_col_offset, arena); + res = obj2ast_int(state, tmp, &end_col_offset, "end_col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11220,7 +11252,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11237,7 +11269,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_arguments(state, tmp, &args, arena); + res = obj2ast_arguments(state, tmp, &args, "args", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11268,7 +11300,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11306,7 +11338,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "decorator_list", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11330,7 +11362,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &returns, arena); + res = obj2ast_expr(state, tmp, &returns, "returns", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11347,7 +11379,8 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_string(state, tmp, &type_comment, arena); + res = obj2ast_string(state, tmp, &type_comment, "type_comment", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11378,7 +11411,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FunctionDef' node")) { goto failed; } - res = obj2ast_type_param(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, "type_params", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11423,7 +11456,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11440,7 +11473,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_arguments(state, tmp, &args, arena); + res = obj2ast_arguments(state, tmp, &args, "args", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11471,7 +11504,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11509,7 +11542,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "decorator_list", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11533,7 +11566,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &returns, arena); + res = obj2ast_expr(state, tmp, &returns, "returns", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11550,7 +11583,8 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_string(state, tmp, &type_comment, arena); + res = obj2ast_string(state, tmp, &type_comment, "type_comment", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11581,7 +11615,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFunctionDef' node")) { goto failed; } - res = obj2ast_type_param(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, "type_params", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11625,7 +11659,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ClassDef' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11656,7 +11690,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ClassDef' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "bases", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11694,7 +11728,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ClassDef' node")) { goto failed; } - res = obj2ast_keyword(state, tmp2, &val, arena); + res = obj2ast_keyword(state, tmp2, &val, "keywords", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11732,7 +11766,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ClassDef' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11770,7 +11804,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ClassDef' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "decorator_list", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11808,7 +11842,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ClassDef' node")) { goto failed; } - res = obj2ast_type_param(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, "type_params", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11846,7 +11880,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Return' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11890,7 +11924,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Delete' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "targets", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11943,7 +11977,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Assign' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "targets", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -11967,7 +12001,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Assign' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -11984,7 +12018,8 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Assign' node")) { goto failed; } - res = obj2ast_string(state, tmp, &type_comment, arena); + res = obj2ast_string(state, tmp, &type_comment, "type_comment", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12016,7 +12051,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'TypeAlias' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &name, arena); + res = obj2ast_expr(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12047,7 +12082,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'TypeAlias' node")) { goto failed; } - res = obj2ast_type_param(state, tmp2, &val, arena); + res = obj2ast_type_param(state, tmp2, &val, "type_params", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12071,7 +12106,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'TypeAlias' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12103,7 +12138,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AugAssign' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &target, arena); + res = obj2ast_expr(state, tmp, &target, "target", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12120,7 +12155,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AugAssign' node")) { goto failed; } - res = obj2ast_operator(state, tmp, &op, arena); + res = obj2ast_operator(state, tmp, &op, "op", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12137,7 +12172,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AugAssign' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12170,7 +12205,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AnnAssign' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &target, arena); + res = obj2ast_expr(state, tmp, &target, "target", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12187,7 +12222,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AnnAssign' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &annotation, arena); + res = obj2ast_expr(state, tmp, &annotation, "annotation", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12204,7 +12239,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AnnAssign' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12221,7 +12256,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AnnAssign' node")) { goto failed; } - res = obj2ast_int(state, tmp, &simple, arena); + res = obj2ast_int(state, tmp, &simple, "simple", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12255,7 +12290,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'For' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &target, arena); + res = obj2ast_expr(state, tmp, &target, "target", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12272,7 +12307,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'For' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &iter, arena); + res = obj2ast_expr(state, tmp, &iter, "iter", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12303,7 +12338,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'For' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12341,7 +12376,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'For' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "orelse", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12365,7 +12400,8 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'For' node")) { goto failed; } - res = obj2ast_string(state, tmp, &type_comment, arena); + res = obj2ast_string(state, tmp, &type_comment, "type_comment", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12399,7 +12435,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFor' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &target, arena); + res = obj2ast_expr(state, tmp, &target, "target", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12416,7 +12452,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFor' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &iter, arena); + res = obj2ast_expr(state, tmp, &iter, "iter", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12447,7 +12483,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFor' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12485,7 +12521,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFor' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "orelse", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12509,7 +12545,8 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncFor' node")) { goto failed; } - res = obj2ast_string(state, tmp, &type_comment, arena); + res = obj2ast_string(state, tmp, &type_comment, "type_comment", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12542,7 +12579,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'While' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &test, arena); + res = obj2ast_expr(state, tmp, &test, "test", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12573,7 +12610,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'While' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12611,7 +12648,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'While' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "orelse", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12650,7 +12687,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'If' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &test, arena); + res = obj2ast_expr(state, tmp, &test, "test", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12681,7 +12718,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'If' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12719,7 +12756,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'If' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "orelse", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12772,7 +12809,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'With' node")) { goto failed; } - res = obj2ast_withitem(state, tmp2, &val, arena); + res = obj2ast_withitem(state, tmp2, &val, "items", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12810,7 +12847,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'With' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12834,7 +12871,8 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'With' node")) { goto failed; } - res = obj2ast_string(state, tmp, &type_comment, arena); + res = obj2ast_string(state, tmp, &type_comment, "type_comment", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12880,7 +12918,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncWith' node")) { goto failed; } - res = obj2ast_withitem(state, tmp2, &val, arena); + res = obj2ast_withitem(state, tmp2, &val, "items", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12918,7 +12956,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncWith' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -12942,7 +12980,8 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'AsyncWith' node")) { goto failed; } - res = obj2ast_string(state, tmp, &type_comment, arena); + res = obj2ast_string(state, tmp, &type_comment, "type_comment", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -12973,7 +13012,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Match' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &subject, arena); + res = obj2ast_expr(state, tmp, &subject, "subject", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13004,7 +13043,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Match' node")) { goto failed; } - res = obj2ast_match_case(state, tmp2, &val, arena); + res = obj2ast_match_case(state, tmp2, &val, "cases", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13042,7 +13081,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Raise' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &exc, arena); + res = obj2ast_expr(state, tmp, &exc, "exc", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13059,7 +13098,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Raise' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &cause, arena); + res = obj2ast_expr(state, tmp, &cause, "cause", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13106,7 +13145,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Try' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13144,7 +13183,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Try' node")) { goto failed; } - res = obj2ast_excepthandler(state, tmp2, &val, arena); + res = obj2ast_excepthandler(state, tmp2, &val, "handlers", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13182,7 +13221,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Try' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "orelse", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13220,7 +13259,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Try' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "finalbody", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13274,7 +13313,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'TryStar' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13312,7 +13351,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'TryStar' node")) { goto failed; } - res = obj2ast_excepthandler(state, tmp2, &val, arena); + res = obj2ast_excepthandler(state, tmp2, &val, "handlers", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13350,7 +13389,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'TryStar' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "orelse", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13388,7 +13427,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'TryStar' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "finalbody", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13426,7 +13465,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Assert' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &test, arena); + res = obj2ast_expr(state, tmp, &test, "test", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13443,7 +13482,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Assert' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &msg, arena); + res = obj2ast_expr(state, tmp, &msg, "msg", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13488,7 +13527,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Import' node")) { goto failed; } - res = obj2ast_alias(state, tmp2, &val, arena); + res = obj2ast_alias(state, tmp2, &val, "names", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13512,7 +13551,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Import' node")) { goto failed; } - res = obj2ast_int(state, tmp, &is_lazy, arena); + res = obj2ast_int(state, tmp, &is_lazy, "is_lazy", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13545,7 +13584,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ImportFrom' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &module, arena); + res = obj2ast_identifier(state, tmp, &module, "module", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13576,7 +13615,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ImportFrom' node")) { goto failed; } - res = obj2ast_alias(state, tmp2, &val, arena); + res = obj2ast_alias(state, tmp2, &val, "names", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13600,7 +13639,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ImportFrom' node")) { goto failed; } - res = obj2ast_int(state, tmp, &level, arena); + res = obj2ast_int(state, tmp, &level, "level", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13617,7 +13656,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ImportFrom' node")) { goto failed; } - res = obj2ast_int(state, tmp, &is_lazy, arena); + res = obj2ast_int(state, tmp, &is_lazy, "is_lazy", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13661,7 +13700,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Global' node")) { goto failed; } - res = obj2ast_identifier(state, tmp2, &val, arena); + res = obj2ast_identifier(state, tmp2, &val, "names", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13712,7 +13751,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Nonlocal' node")) { goto failed; } - res = obj2ast_identifier(state, tmp2, &val, arena); + res = obj2ast_identifier(state, tmp2, &val, "names", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13749,7 +13788,7 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Expr' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13803,8 +13842,8 @@ obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* } int -obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* - arena) +obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, const char* + field, PyArena* arena) { int isinstance; @@ -13819,6 +13858,15 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* *out = NULL; return 0; } + tp = state->expr_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (!isinstance && field != NULL) { + PyErr_Format(PyExc_TypeError, "field '%s' was expecting node of type 'expr', got '%s'", field, _PyType_Name(Py_TYPE(obj))); + return 1; + } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { return -1; } @@ -13831,7 +13879,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'expr' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13848,7 +13896,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'expr' node")) { goto failed; } - res = obj2ast_int(state, tmp, &col_offset, arena); + res = obj2ast_int(state, tmp, &col_offset, "col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13865,7 +13913,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'expr' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_lineno, arena); + res = obj2ast_int(state, tmp, &end_lineno, "end_lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13882,7 +13930,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'expr' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_col_offset, arena); + res = obj2ast_int(state, tmp, &end_col_offset, "end_col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13908,7 +13956,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'BoolOp' node")) { goto failed; } - res = obj2ast_boolop(state, tmp, &op, arena); + res = obj2ast_boolop(state, tmp, &op, "op", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13939,7 +13987,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'BoolOp' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "values", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -13977,7 +14025,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'NamedExpr' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &target, arena); + res = obj2ast_expr(state, tmp, &target, "target", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -13994,7 +14042,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'NamedExpr' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14026,7 +14074,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'BinOp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &left, arena); + res = obj2ast_expr(state, tmp, &left, "left", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14043,7 +14091,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'BinOp' node")) { goto failed; } - res = obj2ast_operator(state, tmp, &op, arena); + res = obj2ast_operator(state, tmp, &op, "op", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14060,7 +14108,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'BinOp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &right, arena); + res = obj2ast_expr(state, tmp, &right, "right", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14091,7 +14139,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'UnaryOp' node")) { goto failed; } - res = obj2ast_unaryop(state, tmp, &op, arena); + res = obj2ast_unaryop(state, tmp, &op, "op", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14108,7 +14156,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'UnaryOp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &operand, arena); + res = obj2ast_expr(state, tmp, &operand, "operand", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14139,7 +14187,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Lambda' node")) { goto failed; } - res = obj2ast_arguments(state, tmp, &args, arena); + res = obj2ast_arguments(state, tmp, &args, "args", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14156,7 +14204,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Lambda' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &body, arena); + res = obj2ast_expr(state, tmp, &body, "body", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14188,7 +14236,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'IfExp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &test, arena); + res = obj2ast_expr(state, tmp, &test, "test", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14205,7 +14253,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'IfExp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &body, arena); + res = obj2ast_expr(state, tmp, &body, "body", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14222,7 +14270,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'IfExp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &orelse, arena); + res = obj2ast_expr(state, tmp, &orelse, "orelse", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14267,7 +14315,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Dict' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "keys", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14305,7 +14353,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Dict' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "values", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14356,7 +14404,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Set' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "elts", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14394,7 +14442,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ListComp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &elt, arena); + res = obj2ast_expr(state, tmp, &elt, "elt", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14425,7 +14473,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'ListComp' node")) { goto failed; } - res = obj2ast_comprehension(state, tmp2, &val, arena); + res = obj2ast_comprehension(state, tmp2, &val, "generators", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14463,7 +14511,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'SetComp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &elt, arena); + res = obj2ast_expr(state, tmp, &elt, "elt", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14494,7 +14542,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'SetComp' node")) { goto failed; } - res = obj2ast_comprehension(state, tmp2, &val, arena); + res = obj2ast_comprehension(state, tmp2, &val, "generators", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14533,7 +14581,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'DictComp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &key, arena); + res = obj2ast_expr(state, tmp, &key, "key", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14550,7 +14598,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'DictComp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14581,7 +14629,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'DictComp' node")) { goto failed; } - res = obj2ast_comprehension(state, tmp2, &val, arena); + res = obj2ast_comprehension(state, tmp2, &val, "generators", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14619,7 +14667,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'GeneratorExp' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &elt, arena); + res = obj2ast_expr(state, tmp, &elt, "elt", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14650,7 +14698,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'GeneratorExp' node")) { goto failed; } - res = obj2ast_comprehension(state, tmp2, &val, arena); + res = obj2ast_comprehension(state, tmp2, &val, "generators", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14687,7 +14735,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Await' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14717,7 +14765,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Yield' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14747,7 +14795,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'YieldFrom' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14779,7 +14827,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Compare' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &left, arena); + res = obj2ast_expr(state, tmp, &left, "left", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14810,7 +14858,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Compare' node")) { goto failed; } - res = obj2ast_cmpop(state, tmp2, &val, arena); + res = obj2ast_cmpop(state, tmp2, &val, "ops", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14848,7 +14896,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Compare' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "comparators", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14887,7 +14935,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Call' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &func, arena); + res = obj2ast_expr(state, tmp, &func, "func", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -14918,7 +14966,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Call' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "args", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14956,7 +15004,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Call' node")) { goto failed; } - res = obj2ast_keyword(state, tmp2, &val, arena); + res = obj2ast_keyword(state, tmp2, &val, "keywords", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -14995,7 +15043,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FormattedValue' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15012,7 +15060,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FormattedValue' node")) { goto failed; } - res = obj2ast_int(state, tmp, &conversion, arena); + res = obj2ast_int(state, tmp, &conversion, "conversion", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15029,7 +15077,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'FormattedValue' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &format_spec, arena); + res = obj2ast_expr(state, tmp, &format_spec, "format_spec", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15063,7 +15111,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Interpolation' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15080,7 +15128,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Interpolation' node")) { goto failed; } - res = obj2ast_constant(state, tmp, &str, arena); + res = obj2ast_constant(state, tmp, &str, "str", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15097,7 +15145,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Interpolation' node")) { goto failed; } - res = obj2ast_int(state, tmp, &conversion, arena); + res = obj2ast_int(state, tmp, &conversion, "conversion", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15114,7 +15162,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Interpolation' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &format_spec, arena); + res = obj2ast_expr(state, tmp, &format_spec, "format_spec", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15159,7 +15207,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'JoinedStr' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "values", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -15210,7 +15258,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'TemplateStr' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "values", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -15248,7 +15296,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Constant' node")) { goto failed; } - res = obj2ast_constant(state, tmp, &value, arena); + res = obj2ast_constant(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15265,7 +15313,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Constant' node")) { goto failed; } - res = obj2ast_string(state, tmp, &kind, arena); + res = obj2ast_string(state, tmp, &kind, "kind", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15297,7 +15345,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Attribute' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15314,7 +15362,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Attribute' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &attr, arena); + res = obj2ast_identifier(state, tmp, &attr, "attr", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15331,7 +15379,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Attribute' node")) { goto failed; } - res = obj2ast_expr_context(state, tmp, &ctx, arena); + res = obj2ast_expr_context(state, tmp, &ctx, "ctx", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15363,7 +15411,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Subscript' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15380,7 +15428,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Subscript' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &slice, arena); + res = obj2ast_expr(state, tmp, &slice, "slice", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15397,7 +15445,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Subscript' node")) { goto failed; } - res = obj2ast_expr_context(state, tmp, &ctx, arena); + res = obj2ast_expr_context(state, tmp, &ctx, "ctx", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15428,7 +15476,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Starred' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15445,7 +15493,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Starred' node")) { goto failed; } - res = obj2ast_expr_context(state, tmp, &ctx, arena); + res = obj2ast_expr_context(state, tmp, &ctx, "ctx", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15476,7 +15524,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Name' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &id, arena); + res = obj2ast_identifier(state, tmp, &id, "id", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15493,7 +15541,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Name' node")) { goto failed; } - res = obj2ast_expr_context(state, tmp, &ctx, arena); + res = obj2ast_expr_context(state, tmp, &ctx, "ctx", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15538,7 +15586,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'List' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "elts", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -15562,7 +15610,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'List' node")) { goto failed; } - res = obj2ast_expr_context(state, tmp, &ctx, arena); + res = obj2ast_expr_context(state, tmp, &ctx, "ctx", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15607,7 +15655,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Tuple' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "elts", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -15631,7 +15679,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Tuple' node")) { goto failed; } - res = obj2ast_expr_context(state, tmp, &ctx, arena); + res = obj2ast_expr_context(state, tmp, &ctx, "ctx", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15663,7 +15711,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Slice' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &lower, arena); + res = obj2ast_expr(state, tmp, &lower, "lower", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15680,7 +15728,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Slice' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &upper, arena); + res = obj2ast_expr(state, tmp, &upper, "upper", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15697,7 +15745,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'Slice' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &step, arena); + res = obj2ast_expr(state, tmp, &step, "step", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -15716,7 +15764,7 @@ obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* int obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* - out, PyArena* arena) + out, const char* field, PyArena* arena) { int isinstance; @@ -15750,8 +15798,8 @@ obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* } int -obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* - arena) +obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, const + char* field, PyArena* arena) { int isinstance; @@ -15778,7 +15826,7 @@ obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* int obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, - PyArena* arena) + const char* field, PyArena* arena) { int isinstance; @@ -15892,8 +15940,8 @@ obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, } int -obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, - PyArena* arena) +obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, const + char* field, PyArena* arena) { int isinstance; @@ -15935,8 +15983,8 @@ obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, } int -obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* - arena) +obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, const + char* field, PyArena* arena) { int isinstance; @@ -16027,7 +16075,7 @@ obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* int obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* - out, PyArena* arena) + out, const char* field, PyArena* arena) { PyObject* tmp = NULL; expr_ty target; @@ -16047,7 +16095,7 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* if (_Py_EnterRecursiveCall(" while traversing 'comprehension' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &target, arena); + res = obj2ast_expr(state, tmp, &target, "target", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16064,7 +16112,7 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* if (_Py_EnterRecursiveCall(" while traversing 'comprehension' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &iter, arena); + res = obj2ast_expr(state, tmp, &iter, "iter", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16095,7 +16143,7 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* if (_Py_EnterRecursiveCall(" while traversing 'comprehension' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "ifs", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -16119,7 +16167,7 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* if (_Py_EnterRecursiveCall(" while traversing 'comprehension' node")) { goto failed; } - res = obj2ast_int(state, tmp, &is_async, arena); + res = obj2ast_int(state, tmp, &is_async, "is_async", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16134,7 +16182,7 @@ obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* int obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* - out, PyArena* arena) + out, const char* field, PyArena* arena) { int isinstance; @@ -16149,6 +16197,15 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* *out = NULL; return 0; } + tp = state->excepthandler_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (!isinstance && field != NULL) { + PyErr_Format(PyExc_TypeError, "field '%s' was expecting node of type 'excepthandler', got '%s'", field, _PyType_Name(Py_TYPE(obj))); + return 1; + } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { return -1; } @@ -16161,7 +16218,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* if (_Py_EnterRecursiveCall(" while traversing 'excepthandler' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16178,7 +16235,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* if (_Py_EnterRecursiveCall(" while traversing 'excepthandler' node")) { goto failed; } - res = obj2ast_int(state, tmp, &col_offset, arena); + res = obj2ast_int(state, tmp, &col_offset, "col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16195,7 +16252,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* if (_Py_EnterRecursiveCall(" while traversing 'excepthandler' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_lineno, arena); + res = obj2ast_int(state, tmp, &end_lineno, "end_lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16212,7 +16269,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* if (_Py_EnterRecursiveCall(" while traversing 'excepthandler' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_col_offset, arena); + res = obj2ast_int(state, tmp, &end_col_offset, "end_col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16239,7 +16296,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* if (_Py_EnterRecursiveCall(" while traversing 'ExceptHandler' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &type, arena); + res = obj2ast_expr(state, tmp, &type, "type", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16256,7 +16313,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* if (_Py_EnterRecursiveCall(" while traversing 'ExceptHandler' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16287,7 +16344,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* if (_Py_EnterRecursiveCall(" while traversing 'ExceptHandler' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -16313,7 +16370,7 @@ obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* int obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, - PyArena* arena) + const char* field, PyArena* arena) { PyObject* tmp = NULL; asdl_arg_seq* posonlyargs; @@ -16350,7 +16407,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'arguments' node")) { goto failed; } - res = obj2ast_arg(state, tmp2, &val, arena); + res = obj2ast_arg(state, tmp2, &val, "posonlyargs", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -16388,7 +16445,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'arguments' node")) { goto failed; } - res = obj2ast_arg(state, tmp2, &val, arena); + res = obj2ast_arg(state, tmp2, &val, "args", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -16412,7 +16469,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'arguments' node")) { goto failed; } - res = obj2ast_arg(state, tmp, &vararg, arena); + res = obj2ast_arg(state, tmp, &vararg, "vararg", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16443,7 +16500,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'arguments' node")) { goto failed; } - res = obj2ast_arg(state, tmp2, &val, arena); + res = obj2ast_arg(state, tmp2, &val, "kwonlyargs", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -16481,7 +16538,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'arguments' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "kw_defaults", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -16505,7 +16562,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'arguments' node")) { goto failed; } - res = obj2ast_arg(state, tmp, &kwarg, arena); + res = obj2ast_arg(state, tmp, &kwarg, "kwarg", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16536,7 +16593,7 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'arguments' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "defaults", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -16558,7 +16615,8 @@ obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, } int -obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) +obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, const char* + field, PyArena* arena) { PyObject* tmp = NULL; identifier arg; @@ -16581,7 +16639,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'arg' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &arg, arena); + res = obj2ast_identifier(state, tmp, &arg, "arg", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16598,7 +16656,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'arg' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &annotation, arena); + res = obj2ast_expr(state, tmp, &annotation, "annotation", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16615,7 +16673,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'arg' node")) { goto failed; } - res = obj2ast_string(state, tmp, &type_comment, arena); + res = obj2ast_string(state, tmp, &type_comment, "type_comment", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16632,7 +16690,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'arg' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16649,7 +16707,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'arg' node")) { goto failed; } - res = obj2ast_int(state, tmp, &col_offset, arena); + res = obj2ast_int(state, tmp, &col_offset, "col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16666,7 +16724,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'arg' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_lineno, arena); + res = obj2ast_int(state, tmp, &end_lineno, "end_lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16683,7 +16741,7 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) if (_Py_EnterRecursiveCall(" while traversing 'arg' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_col_offset, arena); + res = obj2ast_int(state, tmp, &end_col_offset, "end_col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16698,8 +16756,8 @@ obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) } int -obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, - PyArena* arena) +obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, const + char* field, PyArena* arena) { PyObject* tmp = NULL; identifier arg; @@ -16721,7 +16779,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'keyword' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &arg, arena); + res = obj2ast_identifier(state, tmp, &arg, "arg", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16738,7 +16796,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'keyword' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16755,7 +16813,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'keyword' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16772,7 +16830,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'keyword' node")) { goto failed; } - res = obj2ast_int(state, tmp, &col_offset, arena); + res = obj2ast_int(state, tmp, &col_offset, "col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16789,7 +16847,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'keyword' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_lineno, arena); + res = obj2ast_int(state, tmp, &end_lineno, "end_lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16806,7 +16864,7 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'keyword' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_col_offset, arena); + res = obj2ast_int(state, tmp, &end_col_offset, "end_col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16821,8 +16879,8 @@ obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, } int -obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* - arena) +obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, const + char* field, PyArena* arena) { PyObject* tmp = NULL; identifier name; @@ -16844,7 +16902,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'alias' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16861,7 +16919,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'alias' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &asname, arena); + res = obj2ast_identifier(state, tmp, &asname, "asname", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16878,7 +16936,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'alias' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16895,7 +16953,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'alias' node")) { goto failed; } - res = obj2ast_int(state, tmp, &col_offset, arena); + res = obj2ast_int(state, tmp, &col_offset, "col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16912,7 +16970,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'alias' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_lineno, arena); + res = obj2ast_int(state, tmp, &end_lineno, "end_lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16929,7 +16987,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* if (_Py_EnterRecursiveCall(" while traversing 'alias' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_col_offset, arena); + res = obj2ast_int(state, tmp, &end_col_offset, "end_col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16945,7 +17003,7 @@ obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* int obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, - PyArena* arena) + const char* field, PyArena* arena) { PyObject* tmp = NULL; expr_ty context_expr; @@ -16963,7 +17021,7 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'withitem' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &context_expr, arena); + res = obj2ast_expr(state, tmp, &context_expr, "context_expr", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16980,7 +17038,7 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'withitem' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &optional_vars, arena); + res = obj2ast_expr(state, tmp, &optional_vars, "optional_vars", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -16995,7 +17053,7 @@ obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, int obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, - PyArena* arena) + const char* field, PyArena* arena) { PyObject* tmp = NULL; pattern_ty pattern; @@ -17014,7 +17072,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'match_case' node")) { goto failed; } - res = obj2ast_pattern(state, tmp, &pattern, arena); + res = obj2ast_pattern(state, tmp, &pattern, "pattern", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17031,7 +17089,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'match_case' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &guard, arena); + res = obj2ast_expr(state, tmp, &guard, "guard", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17062,7 +17120,7 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'match_case' node")) { goto failed; } - res = obj2ast_stmt(state, tmp2, &val, arena); + res = obj2ast_stmt(state, tmp2, &val, "body", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -17083,8 +17141,8 @@ obj2ast_match_case(struct ast_state *state, PyObject* obj, match_case_ty* out, } int -obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, - PyArena* arena) +obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, const + char* field, PyArena* arena) { int isinstance; @@ -17099,6 +17157,15 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, *out = NULL; return 0; } + tp = state->pattern_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (!isinstance && field != NULL) { + PyErr_Format(PyExc_TypeError, "field '%s' was expecting node of type 'pattern', got '%s'", field, _PyType_Name(Py_TYPE(obj))); + return 1; + } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { return -1; } @@ -17111,7 +17178,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'pattern' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17128,7 +17195,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'pattern' node")) { goto failed; } - res = obj2ast_int(state, tmp, &col_offset, arena); + res = obj2ast_int(state, tmp, &col_offset, "col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17145,7 +17212,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'pattern' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_lineno, arena); + res = obj2ast_int(state, tmp, &end_lineno, "end_lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17162,7 +17229,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'pattern' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_col_offset, arena); + res = obj2ast_int(state, tmp, &end_col_offset, "end_col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17187,7 +17254,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchValue' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &value, arena); + res = obj2ast_expr(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17217,7 +17284,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchSingleton' node")) { goto failed; } - res = obj2ast_constant(state, tmp, &value, arena); + res = obj2ast_constant(state, tmp, &value, "value", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17261,7 +17328,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchSequence' node")) { goto failed; } - res = obj2ast_pattern(state, tmp2, &val, arena); + res = obj2ast_pattern(state, tmp2, &val, "patterns", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -17314,7 +17381,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchMapping' node")) { goto failed; } - res = obj2ast_expr(state, tmp2, &val, arena); + res = obj2ast_expr(state, tmp2, &val, "keys", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -17352,7 +17419,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchMapping' node")) { goto failed; } - res = obj2ast_pattern(state, tmp2, &val, arena); + res = obj2ast_pattern(state, tmp2, &val, "patterns", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -17376,7 +17443,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchMapping' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &rest, arena); + res = obj2ast_identifier(state, tmp, &rest, "rest", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17409,7 +17476,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchClass' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &cls, arena); + res = obj2ast_expr(state, tmp, &cls, "cls", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17440,7 +17507,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchClass' node")) { goto failed; } - res = obj2ast_pattern(state, tmp2, &val, arena); + res = obj2ast_pattern(state, tmp2, &val, "patterns", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -17478,7 +17545,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchClass' node")) { goto failed; } - res = obj2ast_identifier(state, tmp2, &val, arena); + res = obj2ast_identifier(state, tmp2, &val, "kwd_attrs", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -17516,7 +17583,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchClass' node")) { goto failed; } - res = obj2ast_pattern(state, tmp2, &val, arena); + res = obj2ast_pattern(state, tmp2, &val, "kwd_patterns", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -17554,7 +17621,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchStar' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17585,7 +17652,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchAs' node")) { goto failed; } - res = obj2ast_pattern(state, tmp, &pattern, arena); + res = obj2ast_pattern(state, tmp, &pattern, "pattern", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17602,7 +17669,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchAs' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17646,7 +17713,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'MatchOr' node")) { goto failed; } - res = obj2ast_pattern(state, tmp2, &val, arena); + res = obj2ast_pattern(state, tmp2, &val, "patterns", arena); _Py_LeaveRecursiveCall(); Py_DECREF(tmp2); if (res != 0) goto failed; @@ -17672,7 +17739,7 @@ obj2ast_pattern(struct ast_state *state, PyObject* obj, pattern_ty* out, int obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* - out, PyArena* arena) + out, const char* field, PyArena* arena) { int isinstance; @@ -17683,6 +17750,15 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* *out = NULL; return 0; } + tp = state->type_ignore_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (!isinstance && field != NULL) { + PyErr_Format(PyExc_TypeError, "field '%s' was expecting node of type 'type_ignore', got '%s'", field, _PyType_Name(Py_TYPE(obj))); + return 1; + } tp = state->TypeIgnore_type; isinstance = PyObject_IsInstance(obj, tp); if (isinstance == -1) { @@ -17704,7 +17780,7 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* if (_Py_EnterRecursiveCall(" while traversing 'TypeIgnore' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17721,7 +17797,7 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* if (_Py_EnterRecursiveCall(" while traversing 'TypeIgnore' node")) { goto failed; } - res = obj2ast_string(state, tmp, &tag, arena); + res = obj2ast_string(state, tmp, &tag, "tag", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17739,7 +17815,7 @@ obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* int obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, - PyArena* arena) + const char* field, PyArena* arena) { int isinstance; @@ -17754,6 +17830,15 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, *out = NULL; return 0; } + tp = state->type_param_type; + isinstance = PyObject_IsInstance(obj, tp); + if (isinstance == -1) { + return 1; + } + if (!isinstance && field != NULL) { + PyErr_Format(PyExc_TypeError, "field '%s' was expecting node of type 'type_param', got '%s'", field, _PyType_Name(Py_TYPE(obj))); + return 1; + } if (PyObject_GetOptionalAttr(obj, state->lineno, &tmp) < 0) { return -1; } @@ -17766,7 +17851,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } - res = obj2ast_int(state, tmp, &lineno, arena); + res = obj2ast_int(state, tmp, &lineno, "lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17783,7 +17868,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } - res = obj2ast_int(state, tmp, &col_offset, arena); + res = obj2ast_int(state, tmp, &col_offset, "col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17800,7 +17885,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_lineno, arena); + res = obj2ast_int(state, tmp, &end_lineno, "end_lineno", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17817,7 +17902,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'type_param' node")) { goto failed; } - res = obj2ast_int(state, tmp, &end_col_offset, arena); + res = obj2ast_int(state, tmp, &end_col_offset, "end_col_offset", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17844,7 +17929,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'TypeVar' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17861,7 +17946,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'TypeVar' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &bound, arena); + res = obj2ast_expr(state, tmp, &bound, "bound", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17878,7 +17963,8 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'TypeVar' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &default_value, arena); + res = obj2ast_expr(state, tmp, &default_value, "default_value", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17909,7 +17995,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'ParamSpec' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17926,7 +18012,8 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'ParamSpec' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &default_value, arena); + res = obj2ast_expr(state, tmp, &default_value, "default_value", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17957,7 +18044,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'TypeVarTuple' node")) { goto failed; } - res = obj2ast_identifier(state, tmp, &name, arena); + res = obj2ast_identifier(state, tmp, &name, "name", arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -17974,7 +18061,8 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (_Py_EnterRecursiveCall(" while traversing 'TypeVarTuple' node")) { goto failed; } - res = obj2ast_expr(state, tmp, &default_value, arena); + res = obj2ast_expr(state, tmp, &default_value, "default_value", + arena); _Py_LeaveRecursiveCall(); if (res != 0) goto failed; Py_CLEAR(tmp); @@ -18506,7 +18594,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) } mod_ty res = NULL; - if (obj2ast_mod(state, ast, &res, arena) != 0) + if (obj2ast_mod(state, ast, &res, NULL, arena) != 0) return NULL; else return res; diff --git a/Python/asm_trampoline_aarch64.S b/Python/asm_trampoline_aarch64.S new file mode 100644 index 000000000000000..b3aeb728de200c6 --- /dev/null +++ b/Python/asm_trampoline_aarch64.S @@ -0,0 +1,77 @@ +/* + * References: + * - https://developer.arm.com/documentation/101028/0012/5--Feature-test-macros + * - https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst + */ + +#if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 + #define BTI_J hint 36 /* bti j: for jumps, IE br instructions */ + #define BTI_C hint 34 /* bti c: for calls, IE bl instructions */ + #define GNU_PROPERTY_AARCH64_BTI 1 /* bit 0 GNU Notes is for BTI support */ +#else + #define BTI_J + #define BTI_C + #define GNU_PROPERTY_AARCH64_BTI 0 +#endif + +#if defined(__ARM_FEATURE_PAC_DEFAULT) + #if __ARM_FEATURE_PAC_DEFAULT & 1 + #define SIGN_LR hint 25 /* paciasp: sign with the A key */ + #define VERIFY_LR hint 29 /* autiasp: verify with the A key */ + #elif __ARM_FEATURE_PAC_DEFAULT & 2 + #define SIGN_LR hint 27 /* pacibsp: sign with the b key */ + #define VERIFY_LR hint 31 /* autibsp: verify with the b key */ + #endif + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 2 /* bit 1 GNU Notes is for PAC support */ +#else + #define SIGN_LR BTI_C + #define VERIFY_LR + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 +#endif + +#if defined(__ARM_FEATURE_GCS_DEFAULT) && __ARM_FEATURE_GCS_DEFAULT == 1 + #define GNU_PROPERTY_AARCH64_GCS 4 /* bit 2 GNU Notes is for GCS support */ +#else + #define GNU_PROPERTY_AARCH64_GCS 0 +#endif + + .text +#if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) +#if defined(__APPLE__) + .globl __Py_trampoline_func_start +__Py_trampoline_func_start: +#else + .globl _Py_trampoline_func_start +_Py_trampoline_func_start: +#endif + SIGN_LR + stp x29, x30, [sp, -16]! + mov x29, sp + blr x3 + ldp x29, x30, [sp], 16 + VERIFY_LR + ret +#if defined(__APPLE__) + .globl __Py_trampoline_func_end +__Py_trampoline_func_end: +#else + .globl _Py_trampoline_func_end +_Py_trampoline_func_end: + .section .note.GNU-stack,"",@progbits +#endif + +/* Add the BTI, PAC and GCS support to GNU Notes section */ +#if GNU_PROPERTY_AARCH64_BTI != 0 || GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_GCS != 0 + .pushsection .note.gnu.property, "a"; /* Start a new allocatable section */ + .balign 8; /* align it on a byte boundry */ + .long 4; /* size of "GNU\0" */ + .long 0x10; /* size of descriptor */ + .long 0x5; /* NT_GNU_PROPERTY_TYPE_0 */ + .asciz "GNU"; + .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4; /* Four bytes of data */ + .long (GNU_PROPERTY_AARCH64_BTI|GNU_PROPERTY_AARCH64_POINTER_AUTH|GNU_PROPERTY_AARCH64_GCS); /* BTI, PAC or GCS is enabled */ + .long 0; /* padding for 8 byte alignment */ + .popsection; /* end the section */ +#endif +#endif // __aarch64__ && __AARCH64EL__ && !__ILP32__ diff --git a/Python/asm_trampoline_riscv64.S b/Python/asm_trampoline_riscv64.S new file mode 100644 index 000000000000000..6125ba95373f954 --- /dev/null +++ b/Python/asm_trampoline_riscv64.S @@ -0,0 +1,12 @@ + .text + .globl _Py_trampoline_func_start +_Py_trampoline_func_start: + addi sp,sp,-16 + sd ra,8(sp) + jalr a3 + ld ra,8(sp) + addi sp,sp,16 + jr ra + .globl _Py_trampoline_func_end +_Py_trampoline_func_end: + .section .note.GNU-stack,"",@progbits diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline_x86_64.S similarity index 53% rename from Python/asm_trampoline.S rename to Python/asm_trampoline_x86_64.S index 93adae3d99038f8..0e6b11589eafc82 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline_x86_64.S @@ -1,22 +1,12 @@ .text +#ifdef __x86_64__ #if defined(__APPLE__) .globl __Py_trampoline_func_start -#else - .globl _Py_trampoline_func_start -#endif -# The following assembly is equivalent to: -# PyObject * -# trampoline(PyThreadState *ts, _PyInterpreterFrame *f, -# int throwflag, py_evaluator evaluator) -# { -# return evaluator(ts, f, throwflag); -# } -#if defined(__APPLE__) __Py_trampoline_func_start: #else + .globl _Py_trampoline_func_start _Py_trampoline_func_start: #endif -#ifdef __x86_64__ #if defined(__CET__) && (__CET__ & 1) endbr64 #endif @@ -25,24 +15,6 @@ _Py_trampoline_func_start: call *%rcx pop %rbp ret -#endif // __x86_64__ -#if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - // ARM64 little endian, 64bit ABI - // generate with aarch64-linux-gnu-gcc 12.1 - stp x29, x30, [sp, -16]! - mov x29, sp - blr x3 - ldp x29, x30, [sp], 16 - ret -#endif -#ifdef __riscv - addi sp,sp,-16 - sd ra,8(sp) - jalr a3 - ld ra,8(sp) - addi sp,sp,16 - jr ra -#endif #if defined(__APPLE__) .globl __Py_trampoline_func_end __Py_trampoline_func_end: @@ -52,7 +24,7 @@ _Py_trampoline_func_end: .section .note.GNU-stack,"",@progbits #endif # Note for indicating the assembly code supports CET -#if defined(__x86_64__) && defined(__CET__) && (__CET__ & 1) +#if defined(__CET__) && (__CET__ & 1) .section .note.gnu.property,"a" .align 8 .long 1f - 0f @@ -69,4 +41,5 @@ _Py_trampoline_func_end: 3: .align 8 4: +#endif #endif // __x86_64__ diff --git a/Python/assemble.c b/Python/assemble.c index 3df959c36341951..8bcb4c7bf3a9aa4 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -80,9 +80,9 @@ assemble_init(struct assembler *a, int firstlineno) } return SUCCESS; error: - Py_XDECREF(a->a_bytecode); - Py_XDECREF(a->a_linetable); - Py_XDECREF(a->a_except_table); + Py_CLEAR(a->a_bytecode); + Py_CLEAR(a->a_linetable); + Py_CLEAR(a->a_except_table); return ERROR; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 35b30a243318cc3..fa64255be00e75d 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -252,7 +252,6 @@ PyDoc_STRVAR(build_class_doc, Internal helper function used by the class statement."); /*[clinic input] -@permit_long_docstring_body __import__ as builtin___import__ name: object @@ -273,15 +272,16 @@ should be a list of names to emulate ``from name import ...``, or an empty list to emulate ``import name``. When importing a module from a package, note that __import__('A.B', ...) returns package A when fromlist is empty, but its submodule B when -fromlist is not empty. The level argument is used to determine whether to -perform absolute or relative imports: 0 is absolute, while a positive number -is the number of parent directories to search relative to the current module. +fromlist is not empty. The level argument is used to determine whether +to perform absolute or relative imports: 0 is absolute, while a positive +number is the number of parent directories to search relative to the +current module. [clinic start generated code]*/ static PyObject * builtin___import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=4febeda88a0cd245 input=01a3283590eae93a]*/ +/*[clinic end generated code: output=4febeda88a0cd245 input=e3096a230383f72d]*/ { return PyImport_ImportModuleLevelObject(name, globals, locals, fromlist, level); @@ -299,20 +299,26 @@ __lazy_import__ as builtin___lazy_import__ Lazily imports a module. -Returns either the module to be imported or a imp.lazy_module object which -indicates the module to be lazily imported. +Returns either the module to be imported or a imp.lazy_module object +which indicates the module to be lazily imported. [clinic start generated code]*/ static PyObject * builtin___lazy_import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=300f1771094b9e8c input=9394874f340b2948]*/ +/*[clinic end generated code: output=300f1771094b9e8c input=9c85cccd6a885b9b]*/ { PyObject *builtins; PyThreadState *tstate = PyThreadState_GET(); if (globals == NULL) { globals = PyEval_GetGlobals(); + if (globals == NULL) { + PyErr_SetString(PyExc_TypeError, + "__lazy_import__() missing globals " + "when called without a frame"); + return NULL; + } } if (locals == NULL) { locals = globals; @@ -696,8 +702,9 @@ PyDoc_STRVAR(filter_doc, "filter(function, iterable, /)\n\ --\n\ \n\ -Return an iterator yielding those items of iterable for which function(item)\n\ -is true. If function is None, return the items that are true."); +Return an iterator yielding those items of iterable for which\n\ +function(item) is true. If function is None, return the items that\n\ +are true."); PyTypeObject PyFilter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -770,6 +777,7 @@ builtin_format_impl(PyObject *module, PyObject *value, PyObject *format_spec) } /*[clinic input] +@permit_long_summary chr as builtin_chr i: object @@ -780,7 +788,7 @@ Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff. static PyObject * builtin_chr(PyObject *module, PyObject *i) -/*[clinic end generated code: output=d34f25b8035a9b10 input=f919867f0ba2f496]*/ +/*[clinic end generated code: output=d34f25b8035a9b10 input=a9b255f2d2e503f0]*/ { int overflow; long v = PyLong_AsLongAndOverflow(i, &overflow); @@ -804,6 +812,7 @@ builtin_chr(PyObject *module, PyObject *i) /*[clinic input] +@permit_long_summary compile as builtin_compile source: object @@ -818,23 +827,24 @@ compile as builtin_compile Compile source into a code object that can be executed by exec() or eval(). -The source code may represent a Python module, statement or expression. +The source code may represent a Python module, statement or +expression. The filename will be used for run-time error messages. The mode must be 'exec' to compile a module, 'single' to compile a single (interactive) statement, or 'eval' to compile an expression. -The flags argument, if present, controls which future statements influence -the compilation of the code. +The flags argument, if present, controls which future statements +influence the compilation of the code. The dont_inherit argument, if true, stops the compilation inheriting the effects of any future statements in effect in the code calling -compile; if absent or false these statements do influence the compilation, -in addition to any features explicitly specified. +compile; if absent or false these statements do influence the +compilation, in addition to any features explicitly specified. [clinic start generated code]*/ static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, int optimize, PyObject *modname, int feature_version) -/*[clinic end generated code: output=9a0dce1945917a86 input=ddeae1e0253459dc]*/ +/*[clinic end generated code: output=9a0dce1945917a86 input=444c4fe466a97279]*/ { PyObject *source_copy; const char *str; @@ -982,10 +992,10 @@ PyDoc_STRVAR(dir_doc, "dir([object]) -> list of strings\n" "\n" "If called without an argument, return the names in the current scope.\n" -"Else, return an alphabetized list of names comprising (some of) the attributes\n" -"of the given object, and of attributes reachable from it.\n" -"If the object supplies a method named __dir__, it will be used; otherwise\n" -"the default dir() logic is used and returns:\n" +"Else, return an alphabetized list of names comprising (some of) the\n" +"attributes of the given object, and of attributes reachable from it.\n" +"If the object supplies a method named __dir__, it will be used;\n" +"otherwise the default dir() logic is used and returns:\n" " for a module object: the module's attributes.\n" " for a class object: its attributes, and recursively the attributes\n" " of its bases.\n" @@ -1326,9 +1336,11 @@ builtin_getattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(getattr_doc, "getattr(object, name[, default]) -> value\n\ \n\ -Get a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\n\ -When a default argument is given, it is returned when the attribute doesn't\n\ -exist; without it, an exception is raised in that case."); +Get a named attribute from an object.\n\ +\n\ +getattr(x, 'y') is equivalent to x.y.\n\ +When a default argument is given, it is returned when the attribute\n\ +doesn't exist; without it, an exception is raised in that case."); /*[clinic input] @@ -1336,13 +1348,13 @@ globals as builtin_globals Return the dictionary containing the current scope's global variables. -NOTE: Updates to this dictionary *will* affect name lookups in the current -global scope and vice-versa. +NOTE: Updates to this dictionary *will* affect name lookups in the +current global scope and vice-versa. [clinic start generated code]*/ static PyObject * builtin_globals_impl(PyObject *module) -/*[clinic end generated code: output=e5dd1527067b94d2 input=9327576f92bb48ba]*/ +/*[clinic end generated code: output=e5dd1527067b94d2 input=6d725a9b48d1eaeb]*/ { PyObject *globals; if (_PyEval_GetFrame() != NULL) { @@ -1695,8 +1707,8 @@ PyDoc_STRVAR(map_doc, Make an iterator that computes the function using arguments from\n\ each of the iterables. Stops when the shortest iterable is exhausted.\n\ \n\ -If strict is true and one of the arguments is exhausted before the others,\n\ -raise a ValueError."); +If strict is true and one of the arguments is exhausted before the\n\ +others, raise a ValueError."); PyTypeObject PyMap_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -1783,8 +1795,8 @@ builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(next_doc, "next(iterator[, default])\n\ \n\ -Return the next item from the iterator. If default is given and the iterator\n\ -is exhausted, it is returned instead of raising StopIteration."); +Return the next item from the iterator. If default is given and the\n\ +iterator is exhausted, it is returned instead of raising StopIteration."); /*[clinic input] @@ -1907,7 +1919,8 @@ iter(callable, sentinel) -> iterator\n\ \n\ Get an iterator from an object. In the first form, the argument must\n\ supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the sentinel."); +In the second form, the callable is called until it returns the\n\ +sentinel."); /*[clinic input] @@ -2001,14 +2014,15 @@ locals as builtin_locals Return a dictionary containing the current scope's local variables. -NOTE: Whether or not updates to this dictionary will affect name lookups in -the local scope and vice-versa is *implementation dependent* and not -covered by any backwards compatibility guarantees. +NOTE: Whether or not updates to this dictionary will affect name +lookups in the local scope and vice-versa is *implementation +dependent* and not covered by any backwards compatibility +guarantees. [clinic start generated code]*/ static PyObject * builtin_locals_impl(PyObject *module) -/*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ +/*[clinic end generated code: output=b46c94015ce11448 input=989cc75c22167c42]*/ { PyObject *locals; if (_PyEval_GetFrame() != NULL) { @@ -2260,6 +2274,7 @@ builtin_ord(PyObject *module, PyObject *c) /*[clinic input] +@permit_long_summary pow as builtin_pow base: object @@ -2268,14 +2283,14 @@ pow as builtin_pow Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments -Some types, such as ints, are able to use a more efficient algorithm when -invoked using the three argument form. +Some types, such as ints, are able to use a more efficient algorithm +when invoked using the three argument form. [clinic start generated code]*/ static PyObject * builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp, PyObject *mod) -/*[clinic end generated code: output=3ca1538221bbf15f input=435dbd48a12efb23]*/ +/*[clinic end generated code: output=3ca1538221bbf15f input=0cd5c3ecc8003aec]*/ { return PyNumber_Power(base, exp, mod); } @@ -2396,13 +2411,14 @@ Read a string from standard input. The trailing newline is stripped. The prompt string, if given, is printed to standard output without a trailing newline before reading input. -If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError. +If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise +EOFError. On *nix systems, readline is used if available. [clinic start generated code]*/ static PyObject * builtin_input_impl(PyObject *module, PyObject *prompt) -/*[clinic end generated code: output=83db5a191e7a0d60 input=159c46d4ae40977e]*/ +/*[clinic end generated code: output=83db5a191e7a0d60 input=ebb939c954639427]*/ { PyObject *fin = NULL; PyObject *fout = NULL; @@ -2670,13 +2686,14 @@ round as builtin_round Round a number to a given precision in decimal digits. -The return value is an integer if ndigits is omitted or None. Otherwise -the return value has the same type as the number. ndigits may be negative. +The return value is an integer if ndigits is omitted or None. +Otherwise the return value has the same type as the number. ndigits +may be negative. [clinic start generated code]*/ static PyObject * builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) -/*[clinic end generated code: output=ff0d9dd176c02ede input=275678471d7aca15]*/ +/*[clinic end generated code: output=ff0d9dd176c02ede input=bdcb7c67bf4a4320]*/ { PyObject *result; if (ndigits == Py_None) { @@ -2708,8 +2725,8 @@ sorted as builtin_sorted Return a new list containing all items from the iterable in ascending order. -A custom key function can be supplied to customize the sort order, and the -reverse flag can be set to request the result in descending order. +A custom key function can be supplied to customize the sort order, and +the reverse flag can be set to request the result in descending order. [end disabled clinic input]*/ PyDoc_STRVAR(builtin_sorted__doc__, @@ -2843,6 +2860,7 @@ cs_to_double(CompensatedSum total) } /*[clinic input] +@permit_long_summary sum as builtin_sum iterable: object @@ -2852,13 +2870,13 @@ sum as builtin_sum Return the sum of a 'start' value (default: 0) plus an iterable of numbers When the iterable is empty, return the start value. -This function is intended specifically for use with numeric values and may -reject non-numeric types. +This function is intended specifically for use with numeric values and +may reject non-numeric types. [clinic start generated code]*/ static PyObject * builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) -/*[clinic end generated code: output=df758cec7d1d302f input=162b50765250d222]*/ +/*[clinic end generated code: output=df758cec7d1d302f input=d464d57815196b73]*/ { PyObject *result = start; PyObject *temp, *item, *iter; @@ -3094,6 +3112,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) /*[clinic input] +@permit_long_summary isinstance as builtin_isinstance obj: object @@ -3102,15 +3121,15 @@ isinstance as builtin_isinstance Return whether an object is an instance of a class or of a subclass thereof. -A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to -check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B) -or ...`` etc. +A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the +target to check against. This is equivalent to ``isinstance(x, A) or +isinstance(x, B) or ...`` etc. [clinic start generated code]*/ static PyObject * builtin_isinstance_impl(PyObject *module, PyObject *obj, PyObject *class_or_tuple) -/*[clinic end generated code: output=6faf01472c13b003 input=ffa743db1daf7549]*/ +/*[clinic end generated code: output=6faf01472c13b003 input=5d74d547df498f38]*/ { int retval; @@ -3130,15 +3149,15 @@ issubclass as builtin_issubclass Return whether 'cls' is derived from another class or is the same class. -A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to -check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B) -or ...``. +A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the +target to check against. This is equivalent to ``issubclass(x, A) or +issubclass(x, B) or ...``. [clinic start generated code]*/ static PyObject * builtin_issubclass_impl(PyObject *module, PyObject *cls, PyObject *class_or_tuple) -/*[clinic end generated code: output=358412410cd7a250 input=a24b9f3d58c370d6]*/ +/*[clinic end generated code: output=358412410cd7a250 input=a91ce96345a6705d]*/ { int retval; @@ -3368,13 +3387,13 @@ PyDoc_STRVAR(zip_doc, "zip(*iterables, strict=False)\n\ --\n\ \n\ -The zip object yields n-length tuples, where n is the number of iterables\n\ -passed as positional arguments to zip(). The i-th element in every tuple\n\ -comes from the i-th iterable argument to zip(). This continues until the\n\ -shortest argument is exhausted.\n\ +The zip object yields n-length tuples, where n is the number of\n\ +iterables passed as positional arguments to zip(). The i-th element\n\ +in every tuple comes from the i-th iterable argument to zip(). This\n\ +continues until the shortest argument is exhausted.\n\ \n\ -If strict is true and one of the arguments is exhausted before the others,\n\ -raise a ValueError.\n\ +If strict is true and one of the arguments is exhausted before the\n\ +others, raise a ValueError.\n\ \n\ >>> list(zip('abcdefg', range(3), range(4)))\n\ [('a', 0, 0), ('b', 1, 1), ('c', 2, 2)]"); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0277a94ab366329..8afa0be702357de 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -250,6 +250,7 @@ dummy_func( op(_MONITOR_RESUME, (--)) { int err = _Py_call_instrumentation( tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); + RELOAD_STACK(); ERROR_IF(err); if (frame->instr_ptr != this_instr) { /* Instrumentation has jumped */ @@ -1116,7 +1117,8 @@ dummy_func( op(_STORE_SLICE, (v, container, start, stop -- )) { PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); + PyStackRef_AsPyObjectSteal(stop), + Py_None); int err; if (slice == NULL) { err = 1; @@ -1547,6 +1549,7 @@ dummy_func( _PyStackRef executor = frame->localsplus[0]; assert(tstate->current_executor == NULL); if (!PyStackRef_IsNull(executor)) { + assert(PyStackRef_TYPE(executor) == &_PyUOpExecutor_Type); tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); PyStackRef_CLOSE(executor); } @@ -1568,6 +1571,7 @@ dummy_func( DEAD(retval); SAVE_STACK(); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); // GH-99729: We need to unlink the frame *before* clearing it: _PyInterpreterFrame *dying = frame; @@ -1648,6 +1652,8 @@ dummy_func( family(SEND, INLINE_CACHE_ENTRIES_SEND) = { SEND_GEN, + SEND_VIRTUAL, + SEND_ASYNC_GEN, }; specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused, unused -- receiver, unused, unused)) { @@ -1662,9 +1668,8 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } - op(_SEND, (receiver, null_or_index, v -- receiver, null_or_index, retval)) { + tier1 op(_SEND, (receiver, null_or_index, v -- receiver, null_or_index, retval)) { PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && @@ -1683,7 +1688,7 @@ dummy_func( gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } - if (!PyStackRef_IsNull(null_or_index)) { + if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) { _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { @@ -1693,34 +1698,20 @@ dummy_func( DISPATCH(); } retval = item; + DEAD(v); } else { - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - } - else { - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); + PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); + PySendResultPair res = _PyIter_Send(receiver_o, v_o); + if (res.kind == PYGEN_ERROR) { + ERROR_NO_POP(); } - if (retval_o == NULL) { - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - if (matches) { - _PyEval_MonitorRaise(tstate, frame, this_instr); - } - int err = _PyGen_FetchStopIterationValue(&retval_o); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - PyStackRef_CLOSE(v); - ERROR_IF(true); - } + PyStackRef_CLOSE(v); + retval = PyStackRef_FromPyObjectSteal(res.object); + if (res.kind == PYGEN_RETURN) { + JUMPBY(oparg); } - retval = PyStackRef_FromPyObjectSteal(retval_o); } - PyStackRef_CLOSE(v); } macro(SEND) = _SPECIALIZE_SEND + _SEND; @@ -1748,6 +1739,112 @@ dummy_func( _SEND_GEN_FRAME + _PUSH_FRAME; + op(_GUARD_TOS_IS_NONE, (val -- val)) { + EXIT_IF(!PyStackRef_IsNone(val)); + } + + macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; + + op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { + EXIT_IF(PyStackRef_IsNull(nos)); + } + + replaced op(_SEND_VIRTUAL, (iter, null_or_index, none -- iter, null_or_index, next)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + ERROR_NO_POP(); + } + next = none; + DEAD(none); + JUMPBY(oparg); + DISPATCH(); + } + DEAD(none); + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + } + + op(_SEND_VIRTUAL_TIER_TWO, (iter, null_or_index, none -- iter, null_or_index, next)) { + assert(PyStackRef_IsNone(none)); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + ERROR_NO_POP(); + } + next = none; + DEAD(none); + EXIT_IF(true); + } + DEAD(none); + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + } + + macro(SEND_VIRTUAL) = + unused/1 + + _GUARD_TOS_IS_NONE + + _GUARD_NOS_NOT_NULL + + _SEND_VIRTUAL; + + op(_GUARD_3OS_ASYNC_GEN_ASEND, (iter, null_or_index, value -- iter, null_or_index, value)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + EXIT_IF(!PyAsyncGenASend_CheckExact(iter_o)); + } + + replaced op(_SEND_ASYNC_GEN, (iter, null_in, v -- asend, null_out, retval)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyAsyncGenASend_CheckExact(iter_o)); + PyObject *val = PyStackRef_AsPyObjectBorrow(v); + PyObject *retval_o; + + PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); + if (what == PYGEN_ERROR) { + ERROR_NO_POP(); + } + PyStackRef_CLOSE(v); + asend = iter; + DEAD(iter); + null_out = null_in; + DEAD(null_in); + retval = PyStackRef_FromPyObjectSteal(retval_o); + if (what == PYGEN_RETURN) { + JUMPBY(oparg); + } + } + + macro(SEND_ASYNC_GEN) = + unused/1 + + _GUARD_3OS_ASYNC_GEN_ASEND + + _SEND_ASYNC_GEN; + + tier2 op(_SEND_ASYNC_GEN_TIER_TWO, (iter, null_in, v -- asend, null_out, retval)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyAsyncGenASend_CheckExact(iter_o)); + PyObject *val = PyStackRef_AsPyObjectBorrow(v); + PyObject *retval_o; + + PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); + if (what == PYGEN_ERROR) { + ERROR_NO_POP(); + } + PyStackRef_CLOSE(v); + asend = iter; + DEAD(iter); + null_out = null_in; + DEAD(null_in); + retval = PyStackRef_FromPyObjectSteal(retval_o); + EXIT_IF(what == PYGEN_RETURN); + } + op(_YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() @@ -1760,23 +1857,25 @@ dummy_func( _PyStackRef temp = retval; DEAD(retval); SAVE_STACK(); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif RELOAD_STACK(); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); @@ -1793,9 +1892,11 @@ dummy_func( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); if (err) { + RELOAD_STACK(); ERROR_NO_POP(); } if (frame->instr_ptr != this_instr) { + RELOAD_STACK(); next_instr = frame->instr_ptr; DISPATCH(); } @@ -1878,7 +1979,7 @@ dummy_func( inst(LOAD_COMMON_CONSTANT, ( -- value)) { // Keep in sync with _common_constants in opcode.py assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); } inst(LOAD_BUILD_CLASS, ( -- bc)) { @@ -2253,7 +2354,7 @@ dummy_func( assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); DEOPT_IF(res_o == NULL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); @@ -2272,7 +2373,7 @@ dummy_func( DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version); assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); DEOPT_IF(res_o == NULL); #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); @@ -2862,7 +2963,7 @@ dummy_func( assert(keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_value); EXIT_IF(attr_o == NULL); #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); @@ -3757,7 +3858,7 @@ dummy_func( next = item; } - macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; + macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _RECORD_NOS_TYPE + _FOR_ITER; op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); @@ -3772,11 +3873,40 @@ dummy_func( next = item; } + tier2 op(_GUARD_TYPE_ITER, (expected_type/4, iter, null_or_index -- iter, null_or_index)) { + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + EXIT_IF(Py_TYPE(iter_o) != (PyTypeObject *)expected_type); + } + + tier2 op(_ITER_NEXT_INLINE, (iternext_fn/4, iter, null_or_index -- iter, null_or_index, next)) { + assert(sizeof(iternextfunc) == sizeof(uintptr_t)); + volatile iternextfunc iternext_v = (iternextfunc)iternext_fn; + PyObject *item = iternext_v(PyStackRef_AsPyObjectBorrow(iter)); + if (item == NULL) { + if (_PyErr_Occurred(tstate)) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); + _PyErr_Clear(tstate); + } + else { + ERROR_NO_POP(); + } + } + EXIT_IF(true); + } + STAT_INC(FOR_ITER, hit); + next = PyStackRef_FromPyObjectSteal(item); + } + op(_GUARD_NOS_ITER_VIRTUAL, (iter, null_or_index -- iter, null_or_index)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); EXIT_IF(Py_TYPE(iter_o)->_tp_iteritem == NULL); } + op(_GUARD_TOS_NOT_NULL, (null_or_index -- null_or_index)) { + EXIT_IF(PyStackRef_IsNull(null_or_index)); + } + replaced op(_FOR_ITER_VIRTUAL, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); @@ -3797,7 +3927,7 @@ dummy_func( macro(FOR_ITER_VIRTUAL) = unused/1 + // Skip over the counter - _GUARD_NOS_ITER_VIRTUAL + + _GUARD_TOS_NOT_NULL + _FOR_ITER_VIRTUAL; op(_FOR_ITER_VIRTUAL_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { @@ -4140,13 +4270,6 @@ dummy_func( EXIT_IF(!FT_ATOMIC_LOAD_UINT8(ivs->valid)); } - op(_GUARD_KEYS_VERSION, (keys_version/2, owner -- owner)) { - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - EXIT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version); - } - op(_LOAD_ATTR_METHOD_WITH_VALUES, (descr/4, owner -- attr, self)) { assert(oparg & 1); /* Cached method object */ @@ -4163,7 +4286,7 @@ dummy_func( _RECORD_TOS_TYPE + _GUARD_TYPE_VERSION + _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + - _GUARD_KEYS_VERSION + + unused/2 + _LOAD_ATTR_METHOD_WITH_VALUES; op(_LOAD_ATTR_METHOD_NO_DICT, (descr/4, owner -- attr, self)) { @@ -4197,7 +4320,7 @@ dummy_func( _RECORD_TOS_TYPE + _GUARD_TYPE_VERSION + _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + - _GUARD_KEYS_VERSION + + unused/2 + _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; op(_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, (descr/4, owner -- attr)) { @@ -4537,14 +4660,17 @@ dummy_func( assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); DEAD(new_frame); - SYNC_SP(); - _PyFrame_SetStackPointer(frame, stack_pointer); + SAVE_STACK(); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + RELOAD_STACK(); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } @@ -4579,11 +4705,6 @@ dummy_func( EXIT_IF(!PyStackRef_IsNull(null)); } - op(_GUARD_NOS_NOT_NULL, (nos, unused -- nos, unused)) { - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - EXIT_IF(o == NULL); - } - op(_GUARD_THIRD_NULL, (null, unused, unused -- null, unused, unused)) { EXIT_IF(!PyStackRef_IsNull(null)); } @@ -5760,6 +5881,7 @@ dummy_func( gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); @@ -5864,13 +5986,13 @@ dummy_func( } else { original_opcode = _Py_call_instrumentation_line( tstate, frame, this_instr, prev_instr); + RELOAD_STACK(); if (original_opcode < 0) { next_instr = this_instr+1; goto error; } next_instr = frame->instr_ptr; if (next_instr != this_instr) { - SYNC_SP(); DISPATCH(); } } @@ -5881,6 +6003,7 @@ dummy_func( PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } @@ -6361,6 +6484,7 @@ dummy_func( } label(error) { + _PyFrame_StackAssertInvalid(frame); /* Double-check exception status. */ #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { @@ -6387,9 +6511,7 @@ dummy_func( } spilled label(exception_unwind) { - SAVE_STACK(); STOP_TRACING(); - RELOAD_STACK(); /* We can't use frame->instr_ptr here, as RERAISE may have set it */ int offset = INSTR_OFFSET()-1; int level, handler, lasti; @@ -6445,6 +6567,12 @@ dummy_func( } spilled label(exit_unwind) { + assert(_PyErr_Occurred(tstate)); + DTRACE_FUNCTION_RETURN(); + goto exit_unwind_notrace; + } + + spilled label(exit_unwind_notrace) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); @@ -6475,10 +6603,14 @@ dummy_func( } spilled label(start_frame) { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - goto exit_unwind; + goto exit_unwind_notrace; } + DTRACE_FUNCTION_ENTRY(); next_instr = frame->instr_ptr; #ifdef Py_DEBUG int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); @@ -6577,6 +6709,7 @@ dummy_func( error: exception_unwind: exit_unwind: + exit_unwind_notrace: handle_eval_breaker: resume_frame: start_frame: diff --git a/Python/ceval.c b/Python/ceval.c index 28087ba58d4855b..fbea1f67a36f442 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -49,21 +49,9 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) #endif } -void -_Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) -{ - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; -#if _Py_STACK_GROWS_DOWN - if (here_addr < _tstate->c_stack_hard_limit) { -#else - if (here_addr > _tstate->c_stack_hard_limit) { -#endif - Py_FatalError("Unchecked stack overflow."); - } -} - -#if defined(__s390x__) +#if defined(_Py_LINKER_THREAD_STACK_SIZE) +# define Py_C_STACK_SIZE _Py_LINKER_THREAD_STACK_SIZE +#elif defined(__s390x__) # define Py_C_STACK_SIZE 320000 #elif defined(_WIN32) // Don't define Py_C_STACK_SIZE, ask the O/S @@ -278,7 +266,7 @@ PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() - if the stack pointer is between the stack base and c_stack_hard_limit. */ + if the stack pointer is beyond c_stack_soft_limit. */ int _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { @@ -287,16 +275,21 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); #if _Py_STACK_GROWS_DOWN - assert(here_addr >= _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES); if (here_addr < _tstate->c_stack_hard_limit) { - /* Overflowing while handling an overflow. Give up. */ + if (here_addr < _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES) { + // Far out of bounds -- Assume stack switching has occurred + return 0; + } int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; #else - assert(here_addr <= _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES); if (here_addr > _tstate->c_stack_hard_limit) { - /* Overflowing while handling an overflow. Give up. */ + if (here_addr > _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES) { + // Far out of bounds -- Assume stack switching has occurred + return 0; + } int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; #endif + /* Too much stack used to safely raise an exception. Give up. */ char buffer[80]; snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where); Py_FatalError(buffer); @@ -1006,6 +999,14 @@ _Py_assert_within_stack_bounds( abort(); } } +#ifdef _Py_JIT +void +_Py_jit_assert_within_stack_bounds( + _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno +) { + _Py_assert_within_stack_bounds(frame, stack_pointer, "executor_cases.c.h", lineno); +} +#endif #endif int _Py_CheckRecursiveCallPy( @@ -1146,19 +1147,6 @@ _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *index_or_null, int yield_from return PyStackRef_FromPyObjectSteal(iter_o); } -Py_NO_INLINE int -_Py_ReachedRecursionLimit(PyThreadState *tstate) { - uintptr_t here_addr = _Py_get_machine_stack_pointer(); - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - assert(_tstate->c_stack_hard_limit != 0); -#if _Py_STACK_GROWS_DOWN - return here_addr <= _tstate->c_stack_soft_limit; -#else - return here_addr >= _tstate->c_stack_soft_limit; -#endif -} - - #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__) /* * gh-129987: The SLP autovectorizer can cause poor code generation for @@ -1169,11 +1157,43 @@ _Py_ReachedRecursionLimit(PyThreadState *tstate) { * (prior to GCC 9, 40% performance drop), so we have to selectively disable * it. */ -#define DONT_SLP_VECTORIZE __attribute__((optimize ("no-tree-slp-vectorize"))) +#define DONT_SLP_VECTORIZE __attribute__((optimize ("no-tree-slp-vectorize", "no-omit-frame-pointer"))) #else #define DONT_SLP_VECTORIZE #endif +#ifdef WITH_DTRACE +static void +dtrace_function_entry(_PyInterpreterFrame *frame) +{ + const char *filename; + const char *funcname; + int lineno; + + PyCodeObject *code = _PyFrame_GetCode(frame); + filename = PyUnicode_AsUTF8(code->co_filename); + funcname = PyUnicode_AsUTF8(code->co_name); + lineno = PyUnstable_InterpreterFrame_GetLine(frame); + + PyDTrace_FUNCTION_ENTRY(filename, funcname, lineno); +} + +static void +dtrace_function_return(_PyInterpreterFrame *frame) +{ + const char *filename; + const char *funcname; + int lineno; + + PyCodeObject *code = _PyFrame_GetCode(frame); + filename = PyUnicode_AsUTF8(code->co_filename); + funcname = PyUnicode_AsUTF8(code->co_name); + lineno = PyUnstable_InterpreterFrame_GetLine(frame); + + PyDTrace_FUNCTION_RETURN(filename, funcname, lineno); +} +#endif + PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) { @@ -1230,6 +1250,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry.frame.return_offset = 0; #ifdef Py_DEBUG entry.frame.lltrace = 0; + entry.frame.stackpointer_valid = 1; #endif /* Push frame */ entry.frame.previous = tstate->current_frame; @@ -1238,6 +1259,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry.frame.localsplus[0] = PyStackRef_NULL; #ifdef _Py_TIER2 if (tstate->current_executor != NULL) { + assert(Py_TYPE(tstate->current_executor) == &_PyUOpExecutor_Type); entry.frame.localsplus[0] = PyStackRef_FromPyObjectNew(tstate->current_executor); tstate->current_executor = NULL; } @@ -1266,6 +1288,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int next_instr = frame->instr_ptr; monitor_throw(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); @@ -1954,15 +1977,8 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) void _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) { - // Update last_profiled_frame for remote profiler frame caching. // By this point, tstate->current_frame is already set to the parent frame. - // Only update if we're popping the exact frame that was last profiled. - // This avoids corrupting the cache when transient frames (called and returned - // between profiler samples) update last_profiled_frame to addresses the - // profiler never saw. - if (tstate->last_profiled_frame != NULL && tstate->last_profiled_frame == frame) { - tstate->last_profiled_frame = tstate->current_frame; - } + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, tstate->current_frame); if (frame->owner == FRAME_OWNED_BY_THREAD) { clear_thread_frame(tstate, frame); @@ -1988,6 +2004,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, _PyFrame_Initialize(tstate, frame, func, locals, code, 0, previous); if (initialize_locals(tstate, func_obj, frame->localsplus, args, argcount, kwnames)) { assert(frame->owner == FRAME_OWNED_BY_THREAD); + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, tstate->current_frame); clear_thread_frame(tstate, frame); return NULL; } @@ -3027,6 +3044,13 @@ check_lazy_import_compatibility(PyThreadState *tstate, PyObject *globals, return res; } +static int +is_lazy_import_module_level(void) +{ + _PyInterpreterFrame *frame = _PyEval_GetFrame(); + return frame != NULL && frame->f_globals == frame->f_locals; +} + PyObject * _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *name, @@ -3035,21 +3059,24 @@ _PyEval_LazyImportName(PyThreadState *tstate, PyObject *builtins, PyObject *res = NULL; // Check if global policy overrides the local syntax switch (PyImport_GetLazyImportsMode()) { - case PyImport_LAZY_NONE: - lazy = 0; - break; case PyImport_LAZY_ALL: - lazy = 1; + if (!lazy) { + lazy = is_lazy_import_module_level(); + } break; case PyImport_LAZY_NORMAL: break; } - if (!lazy && PyImport_GetLazyImportsMode() != PyImport_LAZY_NONE) { + if (!lazy) { // See if __lazy_modules__ forces this to be lazy. - lazy = check_lazy_import_compatibility(tstate, globals, name, level); - if (lazy < 0) { - return NULL; + // __lazy_modules__ only applies at module level; exec() inside + // functions or classes should remain eager. + if (is_lazy_import_module_level()) { + lazy = check_lazy_import_compatibility(tstate, globals, name, level); + if (lazy < 0) { + return NULL; + } } } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a4e9980589e4a30..0de4477426af358 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -198,7 +198,7 @@ do { \ /* Do interpreter dispatch accounting for tracing and instrumentation */ #define DISPATCH() \ { \ - assert(frame->stackpointer == NULL); \ + _PyFrame_StackAssertInvalid(frame); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ @@ -206,7 +206,7 @@ do { \ #define DISPATCH_NON_TRACING() \ { \ - assert(frame->stackpointer == NULL); \ + _PyFrame_StackAssertInvalid(frame); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO_NON_TRACING(); \ @@ -223,6 +223,7 @@ do { \ do { \ assert(!IS_PEP523_HOOKED(tstate)); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ + _PyFrame_StackPointerValidate(frame); \ assert((NEW_FRAME)->previous == frame); \ frame = tstate->current_frame = (NEW_FRAME); \ CALL_STAT_INC(inlined_py_calls); \ @@ -268,10 +269,10 @@ GETITEM(PyObject *v, Py_ssize_t i) { #if defined(Py_DEBUG) && !defined(_Py_JIT) // This allows temporary stack "overflows", provided it's all in the cache at any point of time. -#define WITHIN_STACK_BOUNDS_IGNORING_CACHE() \ - (frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE())) +#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(F, L) \ + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER || (STACK_LEVEL() >= 0 && (STACK_LEVEL()) <= STACK_SIZE())) #else -#define WITHIN_STACK_BOUNDS_IGNORING_CACHE WITHIN_STACK_BOUNDS +#define ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE ASSERT_WITHIN_STACK_BOUNDS #endif /* Data access macros */ @@ -290,7 +291,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { STAT_INC(opcode, miss); \ STAT_INC((INSTNAME), miss); \ /* The counter is always the first cache entry: */ \ - if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ + if (ADAPTIVE_COUNTER_TRIGGERS(next_instr->cache)) { \ STAT_INC((INSTNAME), deopt); \ } \ } while (0) @@ -328,11 +329,24 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define CONSTS() _PyFrame_GetCode(frame)->co_consts #define NAMES() _PyFrame_GetCode(frame)->co_names +#if defined(WITH_DTRACE) && !defined(Py_BUILD_CORE_MODULE) +static void dtrace_function_entry(_PyInterpreterFrame *); +static void dtrace_function_return(_PyInterpreterFrame *); + #define DTRACE_FUNCTION_ENTRY() \ if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \ dtrace_function_entry(frame); \ } +#define DTRACE_FUNCTION_RETURN() \ + if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \ + dtrace_function_return(frame); \ + } +#else +#define DTRACE_FUNCTION_ENTRY() ((void)0) +#define DTRACE_FUNCTION_RETURN() ((void)0) +#endif + /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ @@ -627,3 +641,6 @@ gen_try_set_executing(PyGenObject *gen) (PyLongObject *)PyStackRef_AsPyObjectBorrow(left), \ (PyLongObject *)PyStackRef_AsPyObjectBorrow(right)); \ } + +#define CALL_TP_ITERITEM_NO_ESCAPE(ITER, INDEX) \ + Py_TYPE(ITER)->_tp_iteritem((ITER), (INDEX)) diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index e6b845cd375d73a..4a38e0df61708c0 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -25,9 +25,10 @@ PyDoc_STRVAR(builtin___import____doc__, "empty list to emulate ``import name``.\n" "When importing a module from a package, note that __import__(\'A.B\', ...)\n" "returns package A when fromlist is empty, but its submodule B when\n" -"fromlist is not empty. The level argument is used to determine whether to\n" -"perform absolute or relative imports: 0 is absolute, while a positive number\n" -"is the number of parent directories to search relative to the current module."); +"fromlist is not empty. The level argument is used to determine whether\n" +"to perform absolute or relative imports: 0 is absolute, while a positive\n" +"number is the number of parent directories to search relative to the\n" +"current module."); #define BUILTIN___IMPORT___METHODDEF \ {"__import__", _PyCFunction_CAST(builtin___import__), METH_FASTCALL|METH_KEYWORDS, builtin___import____doc__}, @@ -120,8 +121,8 @@ PyDoc_STRVAR(builtin___lazy_import____doc__, "\n" "Lazily imports a module.\n" "\n" -"Returns either the module to be imported or a imp.lazy_module object which\n" -"indicates the module to be lazily imported."); +"Returns either the module to be imported or a imp.lazy_module object\n" +"which indicates the module to be lazily imported."); #define BUILTIN___LAZY_IMPORT___METHODDEF \ {"__lazy_import__", _PyCFunction_CAST(builtin___lazy_import__), METH_FASTCALL|METH_KEYWORDS, builtin___lazy_import____doc__}, @@ -339,16 +340,17 @@ PyDoc_STRVAR(builtin_compile__doc__, "\n" "Compile source into a code object that can be executed by exec() or eval().\n" "\n" -"The source code may represent a Python module, statement or expression.\n" +"The source code may represent a Python module, statement or\n" +"expression.\n" "The filename will be used for run-time error messages.\n" "The mode must be \'exec\' to compile a module, \'single\' to compile a\n" "single (interactive) statement, or \'eval\' to compile an expression.\n" -"The flags argument, if present, controls which future statements influence\n" -"the compilation of the code.\n" +"The flags argument, if present, controls which future statements\n" +"influence the compilation of the code.\n" "The dont_inherit argument, if true, stops the compilation inheriting\n" "the effects of any future statements in effect in the code calling\n" -"compile; if absent or false these statements do influence the compilation,\n" -"in addition to any features explicitly specified."); +"compile; if absent or false these statements do influence the\n" +"compilation, in addition to any features explicitly specified."); #define BUILTIN_COMPILE_METHODDEF \ {"compile", _PyCFunction_CAST(builtin_compile), METH_FASTCALL|METH_KEYWORDS, builtin_compile__doc__}, @@ -683,8 +685,8 @@ PyDoc_STRVAR(builtin_globals__doc__, "\n" "Return the dictionary containing the current scope\'s global variables.\n" "\n" -"NOTE: Updates to this dictionary *will* affect name lookups in the current\n" -"global scope and vice-versa."); +"NOTE: Updates to this dictionary *will* affect name lookups in the\n" +"current global scope and vice-versa."); #define BUILTIN_GLOBALS_METHODDEF \ {"globals", (PyCFunction)builtin_globals, METH_NOARGS, builtin_globals__doc__}, @@ -910,9 +912,10 @@ PyDoc_STRVAR(builtin_locals__doc__, "\n" "Return a dictionary containing the current scope\'s local variables.\n" "\n" -"NOTE: Whether or not updates to this dictionary will affect name lookups in\n" -"the local scope and vice-versa is *implementation dependent* and not\n" -"covered by any backwards compatibility guarantees."); +"NOTE: Whether or not updates to this dictionary will affect name\n" +"lookups in the local scope and vice-versa is *implementation\n" +"dependent* and not covered by any backwards compatibility\n" +"guarantees."); #define BUILTIN_LOCALS_METHODDEF \ {"locals", (PyCFunction)builtin_locals, METH_NOARGS, builtin_locals__doc__}, @@ -959,8 +962,8 @@ PyDoc_STRVAR(builtin_pow__doc__, "\n" "Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments\n" "\n" -"Some types, such as ints, are able to use a more efficient algorithm when\n" -"invoked using the three argument form."); +"Some types, such as ints, are able to use a more efficient algorithm\n" +"when invoked using the three argument form."); #define BUILTIN_POW_METHODDEF \ {"pow", _PyCFunction_CAST(builtin_pow), METH_FASTCALL|METH_KEYWORDS, builtin_pow__doc__}, @@ -1136,7 +1139,8 @@ PyDoc_STRVAR(builtin_input__doc__, "The prompt string, if given, is printed to standard output without a\n" "trailing newline before reading input.\n" "\n" -"If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError.\n" +"If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise\n" +"EOFError.\n" "On *nix systems, readline is used if available."); #define BUILTIN_INPUT_METHODDEF \ @@ -1182,8 +1186,9 @@ PyDoc_STRVAR(builtin_round__doc__, "\n" "Round a number to a given precision in decimal digits.\n" "\n" -"The return value is an integer if ndigits is omitted or None. Otherwise\n" -"the return value has the same type as the number. ndigits may be negative."); +"The return value is an integer if ndigits is omitted or None.\n" +"Otherwise the return value has the same type as the number. ndigits\n" +"may be negative."); #define BUILTIN_ROUND_METHODDEF \ {"round", _PyCFunction_CAST(builtin_round), METH_FASTCALL|METH_KEYWORDS, builtin_round__doc__}, @@ -1251,8 +1256,8 @@ PyDoc_STRVAR(builtin_sum__doc__, "Return the sum of a \'start\' value (default: 0) plus an iterable of numbers\n" "\n" "When the iterable is empty, return the start value.\n" -"This function is intended specifically for use with numeric values and may\n" -"reject non-numeric types."); +"This function is intended specifically for use with numeric values and\n" +"may reject non-numeric types."); #define BUILTIN_SUM_METHODDEF \ {"sum", _PyCFunction_CAST(builtin_sum), METH_FASTCALL|METH_KEYWORDS, builtin_sum__doc__}, @@ -1319,9 +1324,9 @@ PyDoc_STRVAR(builtin_isinstance__doc__, "\n" "Return whether an object is an instance of a class or of a subclass thereof.\n" "\n" -"A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to\n" -"check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B)\n" -"or ...`` etc."); +"A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the\n" +"target to check against. This is equivalent to ``isinstance(x, A) or\n" +"isinstance(x, B) or ...`` etc."); #define BUILTIN_ISINSTANCE_METHODDEF \ {"isinstance", _PyCFunction_CAST(builtin_isinstance), METH_FASTCALL, builtin_isinstance__doc__}, @@ -1354,9 +1359,9 @@ PyDoc_STRVAR(builtin_issubclass__doc__, "\n" "Return whether \'cls\' is derived from another class or is the same class.\n" "\n" -"A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to\n" -"check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B)\n" -"or ...``."); +"A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the\n" +"target to check against. This is equivalent to ``issubclass(x, A) or\n" +"issubclass(x, B) or ...``."); #define BUILTIN_ISSUBCLASS_METHODDEF \ {"issubclass", _PyCFunction_CAST(builtin_issubclass), METH_FASTCALL, builtin_issubclass__doc__}, @@ -1382,4 +1387,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f1fc836a63d89826 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=84efa9c5cc737ce5 input=a9049054013a1b77]*/ diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h index 5ed74e6e6ddb6bf..ece7341d65d5fb6 100644 --- a/Python/clinic/context.c.h +++ b/Python/clinic/context.c.h @@ -10,8 +10,8 @@ PyDoc_STRVAR(_contextvars_Context_get__doc__, "\n" "Return the value for `key` if `key` has the value in the context object.\n" "\n" -"If `key` does not exist, return `default`. If `default` is not given,\n" -"return None."); +"If `key` does not exist, return `default`. If `default` is not\n" +"given, return None."); #define _CONTEXTVARS_CONTEXT_GET_METHODDEF \ {"get", _PyCFunction_CAST(_contextvars_Context_get), METH_FASTCALL, _contextvars_Context_get__doc__}, @@ -122,10 +122,12 @@ PyDoc_STRVAR(_contextvars_ContextVar_get__doc__, "\n" "Return a value for the context variable for the current context.\n" "\n" -"If there is no value for the variable in the current context, the method will:\n" -" * return the value of the default argument of the method, if provided; or\n" -" * return the default value for the context variable, if it was created\n" -" with one; or\n" +"If there is no value for the variable in the current context, the\n" +"method will:\n" +" * return the value of the default argument of the method, if\n" +" provided; or\n" +" * return the default value for the context variable, if it was\n" +" created with one; or\n" " * raise a LookupError."); #define _CONTEXTVARS_CONTEXTVAR_GET_METHODDEF \ @@ -160,10 +162,11 @@ PyDoc_STRVAR(_contextvars_ContextVar_set__doc__, "\n" "Call to set a new value for the context variable in the current context.\n" "\n" -"The required value argument is the new value for the context variable.\n" +"The required value argument is the new value for the context\n" +"variable.\n" "\n" -"Returns a Token object that can be used to restore the variable to its previous\n" -"value via the `ContextVar.reset()` method."); +"Returns a Token object that can be used to restore the variable to\n" +"its previous value via the `ContextVar.reset()` method."); #define _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF \ {"set", (PyCFunction)_contextvars_ContextVar_set, METH_O, _contextvars_ContextVar_set__doc__}, @@ -187,8 +190,8 @@ PyDoc_STRVAR(_contextvars_ContextVar_reset__doc__, "\n" "Reset the context variable.\n" "\n" -"The variable is reset to the value it had before the `ContextVar.set()` that\n" -"created the token was used."); +"The variable is reset to the value it had before the\n" +"`ContextVar.set()` that created the token was used."); #define _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF \ {"reset", (PyCFunction)_contextvars_ContextVar_reset, METH_O, _contextvars_ContextVar_reset__doc__}, @@ -256,4 +259,4 @@ token_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=3a04b2fddf24c3e9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=90ec3e4375804e9b input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index de62714ebddafa6..2e4e178b3084063 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -34,8 +34,9 @@ PyDoc_STRVAR(_imp_acquire_lock__doc__, "\n" "Acquires the interpreter\'s import lock for the current thread.\n" "\n" -"This lock should be used by import hooks to ensure thread-safety when importing\n" -"modules. On platforms without threads, this function does nothing."); +"This lock should be used by import hooks to ensure thread-safety when\n" +"importing modules. On platforms without threads, this function does\n" +"nothing."); #define _IMP_ACQUIRE_LOCK_METHODDEF \ {"acquire_lock", (PyCFunction)_imp_acquire_lock, METH_NOARGS, _imp_acquire_lock__doc__}, @@ -664,4 +665,4 @@ _imp__set_lazy_attributes(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=5fa42f580441b3fa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0974db098d601372 input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index 6c00b2b31b007fa..ec0d2eb8a2af543 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -195,8 +195,8 @@ PyDoc_STRVAR(marshal_dumps__doc__, " allow_code\n" " Allow to write code objects.\n" "\n" -"Raise a ValueError exception if value has (or contains an object that has) an\n" -"unsupported type."); +"Raise a ValueError exception if value has (or contains an object that\n" +"has) an unsupported type."); #define MARSHAL_DUMPS_METHODDEF \ {"dumps", _PyCFunction_CAST(marshal_dumps), METH_FASTCALL|METH_KEYWORDS, marshal_dumps__doc__}, @@ -280,8 +280,8 @@ PyDoc_STRVAR(marshal_loads__doc__, " allow_code\n" " Allow to load code objects.\n" "\n" -"If no valid value is found, raise EOFError, ValueError or TypeError. Extra\n" -"bytes in the input are ignored."); +"If no valid value is found, raise EOFError, ValueError or TypeError.\n" +"Extra bytes in the input are ignored."); #define MARSHAL_LOADS_METHODDEF \ {"loads", _PyCFunction_CAST(marshal_loads), METH_FASTCALL|METH_KEYWORDS, marshal_loads__doc__}, @@ -351,4 +351,4 @@ marshal_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -/*[clinic end generated code: output=3e4bfc070a3c78ac input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a574570c3717f60e input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 86e942ec2b8afbc..0c72760f69a4c41 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -761,34 +761,6 @@ sys_getwindowsversion(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(MS_WINDOWS) */ -#if defined(MS_WINDOWS) - -PyDoc_STRVAR(sys__enablelegacywindowsfsencoding__doc__, -"_enablelegacywindowsfsencoding($module, /)\n" -"--\n" -"\n" -"Changes the default filesystem encoding to mbcs:replace.\n" -"\n" -"This is done for consistency with earlier versions of Python. See PEP\n" -"529 for more information.\n" -"\n" -"This is equivalent to defining the PYTHONLEGACYWINDOWSFSENCODING\n" -"environment variable before launching Python."); - -#define SYS__ENABLELEGACYWINDOWSFSENCODING_METHODDEF \ - {"_enablelegacywindowsfsencoding", (PyCFunction)sys__enablelegacywindowsfsencoding, METH_NOARGS, sys__enablelegacywindowsfsencoding__doc__}, - -static PyObject * -sys__enablelegacywindowsfsencoding_impl(PyObject *module); - -static PyObject * -sys__enablelegacywindowsfsencoding(PyObject *module, PyObject *Py_UNUSED(ignored)) -{ - return sys__enablelegacywindowsfsencoding_impl(module); -} - -#endif /* defined(MS_WINDOWS) */ - #if defined(HAVE_DLOPEN) PyDoc_STRVAR(sys_setdlopenflags__doc__, @@ -1395,7 +1367,8 @@ PyDoc_STRVAR(sys__stats_dump__doc__, "\n" "Dump stats to file, and clears the stats.\n" "\n" -"Return False if no statistics were not dumped because stats gathering was off."); +"Return False if no statistics were not dumped because stats gathering\n" +"was off."); #define SYS__STATS_DUMP_METHODDEF \ {"_stats_dump", (PyCFunction)sys__stats_dump, METH_NOARGS, sys__stats_dump__doc__}, @@ -1543,16 +1516,16 @@ PyDoc_STRVAR(sys_remote_exec__doc__, "Executes a file containing Python code in a given remote Python process.\n" "\n" "This function returns immediately, and the code will be executed by the\n" -"target process\'s main thread at the next available opportunity, similarly\n" -"to how signals are handled. There is no interface to determine when the\n" -"code has been executed. The caller is responsible for making sure that\n" -"the file still exists whenever the remote process tries to read it and that\n" -"it hasn\'t been overwritten.\n" +"target process\'s main thread at the next available opportunity,\n" +"similarly to how signals are handled. There is no interface to\n" +"determine when the code has been executed. The caller is responsible\n" +"for making sure that the file still exists whenever the remote process\n" +"tries to read it and that it hasn\'t been overwritten.\n" "\n" -"The remote process must be running a CPython interpreter of the same major\n" -"and minor version as the local process. If either the local or remote\n" -"interpreter is pre-release (alpha, beta, or release candidate) then the\n" -"local and remote interpreters must be the same exact version.\n" +"The remote process must be running a CPython interpreter of the same\n" +"major and minor version as the local process. If either the local or\n" +"remote interpreter is pre-release (alpha, beta, or release candidate)\n" +"then the local and remote interpreters must be the same exact version.\n" "\n" "Args:\n" " pid (int): The process ID of the target Python process.\n" @@ -1914,8 +1887,8 @@ PyDoc_STRVAR(sys_set_lazy_imports__doc__, "\n" "The mode parameter must be one of the following strings:\n" "- \"all\": All top-level imports become potentially lazy\n" -"- \"none\": All lazy imports are suppressed (even explicitly marked ones)\n" -"- \"normal\": Only explicitly marked imports (with \'lazy\' keyword) are lazy\n" +"- \"normal\": Only explicitly marked imports (with \'lazy\' keyword) are\n" +" lazy\n" "\n" "In addition to the mode, lazy imports can be controlled via the filter\n" "provided to sys.set_lazy_imports_filter"); @@ -1979,7 +1952,6 @@ PyDoc_STRVAR(sys_get_lazy_imports__doc__, "Gets the global lazy imports mode.\n" "\n" "Returns \"all\" if all top level imports are potentially lazy.\n" -"Returns \"none\" if all explicitly marked lazy imports are suppressed.\n" "Returns \"normal\" if only explicitly marked imports are lazy."); #define SYS_GET_LAZY_IMPORTS_METHODDEF \ @@ -2082,10 +2054,6 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #define SYS_GETWINDOWSVERSION_METHODDEF #endif /* !defined(SYS_GETWINDOWSVERSION_METHODDEF) */ -#ifndef SYS__ENABLELEGACYWINDOWSFSENCODING_METHODDEF - #define SYS__ENABLELEGACYWINDOWSFSENCODING_METHODDEF -#endif /* !defined(SYS__ENABLELEGACYWINDOWSFSENCODING_METHODDEF) */ - #ifndef SYS_SETDLOPENFLAGS_METHODDEF #define SYS_SETDLOPENFLAGS_METHODDEF #endif /* !defined(SYS_SETDLOPENFLAGS_METHODDEF) */ @@ -2121,4 +2089,4 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=e8333fe10c01ae66 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8a4482f9c5c493e5 input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index 0bde56c0ac662e1..6d1ae651fa00058 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -10,6 +10,8 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_codecs.h" // export _PyCodec_LookupTextEncoding() +#include "pycore_initconfig.h" // _Py_DumpPathConfig() #include "pycore_interp.h" // PyInterpreterState.codec_search_path #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -1685,6 +1687,8 @@ _PyCodec_InitRegistry(PyInterpreterState *interp) // search functions, so this is done after everything else is initialized. PyObject *mod = PyImport_ImportModule("encodings"); if (mod == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + _Py_DumpPathConfig(tstate); return PyStatus_Error("Failed to import encodings module"); } Py_DECREF(mod); diff --git a/Python/codegen.c b/Python/codegen.c index a77451152c60394..e65c308617df5ee 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -733,14 +733,8 @@ codegen_setup_annotations_scope(compiler *c, location loc, } static int -codegen_leave_annotations_scope(compiler *c, location loc) +codegen_rename_annotations_format_param(PyCodeObject *co) { - ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); - PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); - if (co == NULL) { - return ERROR; - } - // We want the parameter to __annotate__ to be named "format" in the // signature shown by inspect.signature(), but we need to use a // different name (.format) in the symtable; if the name @@ -749,19 +743,16 @@ codegen_leave_annotations_scope(compiler *c, location loc) // co->co_localsplusnames = ("format", *co->co_localsplusnames[1:]) const Py_ssize_t size = PyObject_Size(co->co_localsplusnames); if (size == -1) { - Py_DECREF(co); return ERROR; } PyObject *new_names = PyTuple_New(size); if (new_names == NULL) { - Py_DECREF(co); return ERROR; } PyTuple_SET_ITEM(new_names, 0, Py_NewRef(&_Py_ID(format))); for (int i = 1; i < size; i++) { PyObject *item = PyTuple_GetItem(co->co_localsplusnames, i); if (item == NULL) { - Py_DECREF(co); Py_DECREF(new_names); return ERROR; } @@ -769,6 +760,22 @@ codegen_leave_annotations_scope(compiler *c, location loc) PyTuple_SET_ITEM(new_names, i, item); } Py_SETREF(co->co_localsplusnames, new_names); + return SUCCESS; +} + +static int +codegen_leave_annotations_scope(compiler *c, location loc) +{ + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); + PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); + if (co == NULL) { + return ERROR; + } + + if (codegen_rename_annotations_format_param(co) < 0) { + Py_DECREF(co); + return ERROR; + } _PyCompile_ExitScope(c); int ret = codegen_make_closure(c, loc, co, 0); @@ -1269,6 +1276,10 @@ codegen_type_param_bound_or_default(compiler *c, expr_ty e, if (co == NULL) { return ERROR; } + if (codegen_rename_annotations_format_param(co) < 0) { + Py_DECREF(co); + return ERROR; + } int ret = codegen_make_closure(c, LOC(e), co, MAKE_FUNCTION_DEFAULTS); Py_DECREF(co); RETURN_IF_ERROR(ret); @@ -1770,6 +1781,10 @@ codegen_typealias_body(compiler *c, stmt_ty s) if (co == NULL) { return ERROR; } + if (codegen_rename_annotations_format_param(co) < 0) { + Py_DECREF(co); + return ERROR; + } int ret = codegen_make_closure(c, loc, co, MAKE_FUNCTION_DEFAULTS); Py_DECREF(co); RETURN_IF_ERROR(ret); @@ -2156,7 +2171,7 @@ codegen_for(compiler *c, stmt_ty s) USE_LABEL(c, cleanup); /* It is important for instrumentation that the `END_FOR` comes first. * Iteration over a generator will jump to the first of these instructions, - * but a non-generator will jump to a later instruction. + * but a non-generator will jump to the second instruction. */ ADDOP(c, NO_LOCATION, END_FOR); ADDOP(c, NO_LOCATION, POP_ITER); @@ -3953,22 +3968,45 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) if (! (func->kind == Name_kind && asdl_seq_LEN(args) == 1 && - asdl_seq_LEN(kwds) == 0 && - asdl_seq_GET(args, 0)->kind == GeneratorExp_kind)) + asdl_seq_LEN(kwds) == 0)) { return 0; } - expr_ty generator_exp = asdl_seq_GET(args, 0); - PySTEntryObject *generator_entry = _PySymtable_Lookup(SYMTABLE(c), (void *)generator_exp); + location loc = LOC(func); + + expr_ty arg_expr = asdl_seq_GET(args, 0); + + if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "frozenset") + && (arg_expr->kind == Set_kind || arg_expr->kind == SetComp_kind)) { + NEW_JUMP_TARGET_LABEL(c, skip_optimization); + + ADDOP_I(c, loc, COPY, 1); + ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_BUILTIN_FROZENSET); + ADDOP_COMPARE(c, loc, Is); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, skip_optimization); + ADDOP(c, loc, POP_TOP); + + VISIT(c, expr, arg_expr); + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_BUILD_FROZENSET); + + ADDOP_JUMP(c, loc, JUMP, end); + + USE_LABEL(c, skip_optimization); + return 1; + } + + if (arg_expr->kind != GeneratorExp_kind) { + return 0; + } + + PySTEntryObject *generator_entry = _PySymtable_Lookup(SYMTABLE(c), (void *)arg_expr); if (generator_entry->ste_coroutine) { Py_DECREF(generator_entry); return 0; } Py_DECREF(generator_entry); - location loc = LOC(func); - int optimized = 0; NEW_JUMP_TARGET_LABEL(c, skip_optimization); @@ -3994,6 +4032,9 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) else if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "set")) { const_oparg = CONSTANT_BUILTIN_SET; } + else if (_PyUnicode_EqualToASCIIString(func->v.Name.id, "frozenset")) { + const_oparg = CONSTANT_BUILTIN_FROZENSET; + } if (const_oparg != -1) { ADDOP_I(c, loc, COPY, 1); // the function ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, const_oparg); @@ -4003,10 +4044,10 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) if (const_oparg == CONSTANT_BUILTIN_TUPLE || const_oparg == CONSTANT_BUILTIN_LIST) { ADDOP_I(c, loc, BUILD_LIST, 0); - } else if (const_oparg == CONSTANT_BUILTIN_SET) { + } else if (const_oparg == CONSTANT_BUILTIN_SET || const_oparg == CONSTANT_BUILTIN_FROZENSET) { ADDOP_I(c, loc, BUILD_SET, 0); } - VISIT(c, expr, generator_exp); + VISIT(c, expr, arg_expr); NEW_JUMP_TARGET_LABEL(c, loop); NEW_JUMP_TARGET_LABEL(c, cleanup); @@ -4017,7 +4058,7 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) if (const_oparg == CONSTANT_BUILTIN_TUPLE || const_oparg == CONSTANT_BUILTIN_LIST) { ADDOP_I(c, loc, LIST_APPEND, 3); ADDOP_JUMP(c, loc, JUMP, loop); - } else if (const_oparg == CONSTANT_BUILTIN_SET) { + } else if (const_oparg == CONSTANT_BUILTIN_SET || const_oparg == CONSTANT_BUILTIN_FROZENSET) { ADDOP_I(c, loc, SET_ADD, 3); ADDOP_JUMP(c, loc, JUMP, loop); } @@ -4029,7 +4070,8 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) ADDOP(c, NO_LOCATION, POP_ITER); if (const_oparg != CONSTANT_BUILTIN_TUPLE && const_oparg != CONSTANT_BUILTIN_LIST && - const_oparg != CONSTANT_BUILTIN_SET) { + const_oparg != CONSTANT_BUILTIN_SET && + const_oparg != CONSTANT_BUILTIN_FROZENSET) { ADDOP_LOAD_CONST(c, loc, initial_res == Py_True ? Py_False : Py_True); } ADDOP_JUMP(c, loc, JUMP, end); @@ -4044,6 +4086,9 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) } else if (const_oparg == CONSTANT_BUILTIN_SET) { // result is already a set } + else if (const_oparg == CONSTANT_BUILTIN_FROZENSET) { + ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_BUILD_FROZENSET); + } else { ADDOP_LOAD_CONST(c, loc, initial_res); } diff --git a/Python/compile.c b/Python/compile.c index eb9fc827bea40a8..e223ef42a42e22b 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -893,12 +893,15 @@ compiler_mod(compiler *c, mod_ty mod) { PyCodeObject *co = NULL; int addNone = mod->kind != Expression_kind; + assert(c->u == NULL); if (compiler_codegen(c, mod) < 0) { goto finally; } co = _PyCompile_OptimizeAndAssemble(c, addNone); finally: - _PyCompile_ExitScope(c); + if (c->u != NULL) { + _PyCompile_ExitScope(c); + } return co; } diff --git a/Python/context.c b/Python/context.c index 62b582f271ffe57..4678054ff3ad743 100644 --- a/Python/context.c +++ b/Python/context.c @@ -362,6 +362,9 @@ PyContextVar_Set(PyObject *ovar, PyObject *val) Py_XINCREF(old_val); PyContextToken *tok = token_new(ctx, var, old_val); Py_XDECREF(old_val); + if (tok == NULL) { + return NULL; + } if (contextvar_set(var, val)) { Py_DECREF(tok); @@ -618,6 +621,7 @@ context_tp_contains(PyObject *op, PyObject *key) /*[clinic input] +@permit_long_summary _contextvars.Context.get key: object default: object = None @@ -625,14 +629,14 @@ _contextvars.Context.get Return the value for `key` if `key` has the value in the context object. -If `key` does not exist, return `default`. If `default` is not given, -return None. +If `key` does not exist, return `default`. If `default` is not +given, return None. [clinic start generated code]*/ static PyObject * _contextvars_Context_get_impl(PyContext *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=0c54aa7664268189 input=c8eeb81505023995]*/ +/*[clinic end generated code: output=0c54aa7664268189 input=d669a0d56fabb0a5]*/ { if (context_check_key_type(key)) { return NULL; @@ -1006,23 +1010,24 @@ contextvar_tp_repr(PyObject *op) /*[clinic input] -@permit_long_docstring_body _contextvars.ContextVar.get default: object = NULL / Return a value for the context variable for the current context. -If there is no value for the variable in the current context, the method will: - * return the value of the default argument of the method, if provided; or - * return the default value for the context variable, if it was created - with one; or +If there is no value for the variable in the current context, the +method will: + * return the value of the default argument of the method, if + provided; or + * return the default value for the context variable, if it was + created with one; or * raise a LookupError. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_get_impl(PyContextVar *self, PyObject *default_value) -/*[clinic end generated code: output=0746bd0aa2ced7bf input=da66664d5d0af4ad]*/ +/*[clinic end generated code: output=0746bd0aa2ced7bf input=83814c6aef4a9fe3]*/ { PyObject *val; if (PyContextVar_Get((PyObject *)self, default_value, &val) < 0) { @@ -1038,41 +1043,41 @@ _contextvars_ContextVar_get_impl(PyContextVar *self, PyObject *default_value) } /*[clinic input] -@permit_long_docstring_body +@permit_long_summary _contextvars.ContextVar.set value: object / Call to set a new value for the context variable in the current context. -The required value argument is the new value for the context variable. +The required value argument is the new value for the context +variable. -Returns a Token object that can be used to restore the variable to its previous -value via the `ContextVar.reset()` method. +Returns a Token object that can be used to restore the variable to +its previous value via the `ContextVar.reset()` method. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_set_impl(PyContextVar *self, PyObject *value) -/*[clinic end generated code: output=1b562d35cc79c806 input=73ebbbfc7c98f6cd]*/ +/*[clinic end generated code: output=1b562d35cc79c806 input=04ef8dcd810f5be6]*/ { return PyContextVar_Set((PyObject *)self, value); } /*[clinic input] -@permit_long_docstring_body _contextvars.ContextVar.reset token: object / Reset the context variable. -The variable is reset to the value it had before the `ContextVar.set()` that -created the token was used. +The variable is reset to the value it had before the +`ContextVar.set()` that created the token was used. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_reset_impl(PyContextVar *self, PyObject *token) -/*[clinic end generated code: output=3205d2bdff568521 input=b8bc514a9245242a]*/ +/*[clinic end generated code: output=3205d2bdff568521 input=dd33cfcb18c00e37]*/ { if (!PyContextToken_CheckExact(token)) { PyErr_Format(PyExc_TypeError, @@ -1098,7 +1103,8 @@ static PyMethodDef PyContextVar_methods[] = { _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("ContextVars are generic over the type of their contained values")}, {NULL, NULL} }; @@ -1264,7 +1270,8 @@ token_exit_impl(PyContextToken *self, PyObject *type, PyObject *val, static PyMethodDef PyContextTokenType_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Tokens are generic over the same type as the ContextVar which created them.")}, TOKEN_ENTER_METHODDEF TOKEN_EXIT_METHODDEF {NULL} diff --git a/Python/critical_section.c b/Python/critical_section.c index 98e23eda7cdd77e..dbee6f236a73bea 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -72,6 +72,22 @@ _PyCriticalSection2_BeginSlow(PyThreadState *tstate, PyCriticalSection2 *c, PyMu c->_cs_base._cs_prev = 0; return; } + // Same optimization as in _PyCriticalSection_BeginSlow: skip locking when + // recursively acquiring the same locks. + if (tstate->critical_section && + tstate->critical_section & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *prev2 = (PyCriticalSection2 *) + untag_critical_section(tstate->critical_section); + assert((uintptr_t)m1 < (uintptr_t)m2); + assert((uintptr_t)prev2->_cs_base._cs_mutex < + (uintptr_t)prev2->_cs_mutex2); + if (prev2->_cs_base._cs_mutex == m1 && prev2->_cs_mutex2 == m2) { + c->_cs_base._cs_mutex = NULL; + c->_cs_mutex2 = NULL; + c->_cs_base._cs_prev = 0; + return; + } + } c->_cs_base._cs_mutex = NULL; c->_cs_mutex2 = NULL; c->_cs_base._cs_prev = tstate->critical_section; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 6b489bf03f86ecd..ed77c1be646e275 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1689,6 +1689,7 @@ _PyXI_NewExcInfo(PyObject *exc) } _PyXI_excinfo *info = PyMem_RawCalloc(1, sizeof(_PyXI_excinfo)); if (info == NULL) { + PyErr_NoMemory(); return NULL; } const char *failure; @@ -1709,6 +1710,7 @@ _PyXI_NewExcInfo(PyObject *exc) void _PyXI_FreeExcInfo(_PyXI_excinfo *info) { + assert(info != NULL); _PyXI_excinfo_clear(info); PyMem_RawFree(info); } diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 2e1455fbe232f49..0ff88ad330fd09c 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -38,6 +38,7 @@ const char *_PyImport_DynLoadFiletab[] = { #ifdef __CYGWIN__ + "." SOABI ".dll", ".dll", #else /* !__CYGWIN__ */ "." SOABI ".so", diff --git a/Python/dynload_win.c b/Python/dynload_win.c index de9b0a77817a633..1c2544e94160ac2 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -151,11 +151,16 @@ static char *GetPythonImport (HINSTANCE hModule) to this python DLL is loaded, not a python3.dll that might be on the path by chance. Return whether the DLL was found. + On free-threaded builds, PY3_DLLNAME is undefined and this is a no-op. + _Py_CheckPython3t will check for python3t.dll in that case. */ extern HMODULE PyWin_DLLhModule; static int _Py_CheckPython3(void) { +#ifndef PY3_DLLNAME + return 1; +#else static int python3_checked = 0; static HANDLE hPython3; #define MAXPATHLEN 512 @@ -169,10 +174,11 @@ _Py_CheckPython3(void) use that DLL */ if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, py3path, MAXPATHLEN)) { wchar_t *p = wcsrchr(py3path, L'\\'); + if (p) { - wcscpy(p + 1, PY3_DLLNAME); + wcscpy(p + 1, PY3_DLLNAME L".dll"); hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); - if (hPython3 != NULL) { + if (hPython3) { return 1; } } @@ -180,7 +186,7 @@ _Py_CheckPython3(void) /* If we can locate python3.dll in our application dir, use that DLL */ - hPython3 = LoadLibraryExW(PY3_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); + hPython3 = LoadLibraryExW(PY3_DLLNAME L".dll", NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); if (hPython3 != NULL) { return 1; } @@ -192,13 +198,78 @@ _Py_CheckPython3(void) assert(config->prefix); if (config->prefix) { wcscpy_s(py3path, MAXPATHLEN, config->prefix); - if (py3path[0] && _Py_add_relfile(py3path, L"DLLs\\" PY3_DLLNAME, MAXPATHLEN) >= 0) { + if (py3path[0] && _Py_add_relfile(py3path, L"DLLs\\" PY3_DLLNAME L".dll", MAXPATHLEN) >= 0) { hPython3 = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); } } return hPython3 != NULL; #undef MAXPATHLEN +#endif /* PY3_DLLNAME */ +} + +/* To support extensions that can load with both abi3 and abi3t, we also need to + * preload python3t.dll. Due to 3.15 still supporting intermingled layouts, the + * check is a bit more complicated on that version as we need to try loading + * from a subdirectory first in case the adjacent python3t.dll is meant for + * python315t.dll (and we are python315.dll). + */ +static int +_Py_CheckPython3t(void) +{ +#ifndef ABI3T_DLLNAME + return 1; +#else +#if defined(PY3_DLLNAME) && PY_MAJOR_VERSION==3 && PY_MINOR_VERSION==15 + /* GIL-enabled builds of 3.15 might have a python3t.dll adjacent that is for + a free-threaded build. So we first check in a subdirectory in that case. + If it's not there, we'll look adjacent. */ + #define ABI3T_COMPAT_DLLNAME L"abi3t-compat\\" ABI3T_DLLNAME L".dll" +#endif + static int python3t_checked = 0; + static HANDLE hPython3t; + #define MAXPATHLEN 512 + wchar_t py3path[MAXPATHLEN+1]; + if (python3t_checked) { + return hPython3t != NULL; + } + python3t_checked = 1; + + /* If there is a python3t.dll [in the abi3t-compat dir] next to the + python3y.dll, use that DLL */ + if (PyWin_DLLhModule && GetModuleFileNameW(PyWin_DLLhModule, py3path, MAXPATHLEN)) { + wchar_t *p = wcsrchr(py3path, L'\\'); + + if (p) { +#ifdef ABI3T_COMPAT_DLLNAME + wcscpy(p + 1, ABI3T_COMPAT_DLLNAME); + hPython3t = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + if (hPython3t == NULL) +#endif + { + wcscpy(p + 1, ABI3T_DLLNAME L".dll"); + hPython3t = LoadLibraryExW(py3path, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + } + if (hPython3t) { + return 1; + } + } + } + + /* If we can locate python3.dll in our application dir, + use that DLL */ +#ifdef ABI3T_COMPAT_DLLNAME + hPython3t = LoadLibraryExW(ABI3T_COMPAT_DLLNAME, NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); + #undef ABI3T_COMPAT_DLLNAME + if (hPython3t == NULL) +#endif + { + hPython3t = LoadLibraryExW(ABI3T_DLLNAME L".dll", NULL, LOAD_LIBRARY_SEARCH_APPLICATION_DIR); + } + return hPython3t != NULL; + #undef MAXPATHLEN +#endif /* ABI3T_DLLNAME */ } + #endif /* Py_ENABLE_SHARED */ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, @@ -210,6 +281,7 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, #ifdef Py_ENABLE_SHARED _Py_CheckPython3(); + _Py_CheckPython3t(); #endif /* Py_ENABLE_SHARED */ wchar_t *wpathname = PyUnicode_AsWideCharString(pathname, NULL); diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index 98ee44276e53e03..b0c370ced36cc33 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -23,24 +23,27 @@ int __syscall_getuid32(void) { return __syscall_getuid32_js(); } -EM_JS(int, __syscall_umask_js, (int mask), { - // If we're in node and we can, call native process.umask() +// Emscripten's syscall layer tracks the umask in SYSCALLS.currentUmask and +// applies it itself when creating files and directories. We mount the real +// filesystem via NODEFS, which applies proces.umask() to everything as well. To +// avoid masking the mode twice, read and zero out process umask at startup, +// and store it as emscripten's umask. +EM_JS(void, __syscall_init_umask_js, (void), { if (ENVIRONMENT_IS_NODE) { try { - return process.umask(mask); - } catch(e) { - // oops... - // NodeJS docs: "In Worker threads, process.umask(mask) will throw an exception." - // umask docs: "This system call always succeeds" - return 0; + // process.umask(0) returns the previous umask and sets it to 0. + SYSCALLS.currentUmask = process.umask(0); + } catch (e) { + // NodeJS docs: "In Worker threads, process.umask(mask) will throw an + // exception." In that case just keep emscripten's default umask. } } - // Fall back to the stub case of returning 0. - return 0; }) -int __syscall_umask(int mask) { - return __syscall_umask_js(mask); +EM_JS_DEPS(__syscall_init_umask, "$SYSCALLS"); + +__attribute__((constructor)) void __syscall_init_umask(void) { + __syscall_init_umask_js(); } #include <wasi/api.h> @@ -290,6 +293,26 @@ int __syscall_poll(intptr_t fds, int nfds, int timeout) { return __block_for_int(p); } + +// Workaround for an Emscripten bug: getentropy(buffer, 1) returns the single +// byte of entropy as the return code. Fixed upstream by +// emscripten-core/emscripten#27122 +int __real_getentropy(void*, size_t); + +int __wrap_getentropy(void *buffer, size_t len) { + if (len != 1) { + return __real_getentropy(buffer, len); + } + // Length is 1. Workaround is to get two bytes of entropy and write the + // first one into the original target buffer. + uint8_t tmp[2]; + int ret = __real_getentropy(tmp, 2); + if (ret == 0) { + *(uint8_t *)buffer = tmp[0]; + } + return ret; +} + #include <sys/ioctl.h> int syscall_ioctl_orig(int fd, int request, void* varargs) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b670ff3e7661558..63b77c9e1527787 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10,37 +10,37 @@ case _NOP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _NOP_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _NOP_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _NOP_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -48,16 +48,17 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_PERIODIC_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -66,7 +67,7 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -74,12 +75,13 @@ case _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); oparg = CURRENT_OPARG(); if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -89,7 +91,7 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -97,7 +99,7 @@ case _RESUME_CHECK_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); @@ -123,13 +125,13 @@ } #endif SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RESUME_CHECK_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -160,13 +162,13 @@ #endif _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RESUME_CHECK_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #if defined(__EMSCRIPTEN__) @@ -202,13 +204,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RESUME_CHECK_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -249,7 +251,7 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -257,30 +259,31 @@ case _LOAD_FAST_CHECK_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = CURRENT_OPARG(); _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } value = PyStackRef_DUP(value_s); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_CHECK_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -290,11 +293,12 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -302,13 +306,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_CHECK_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -320,11 +324,12 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -333,13 +338,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_0_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); @@ -347,13 +352,13 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_0_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 0; @@ -363,13 +368,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_0_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -381,13 +386,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_1_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); @@ -395,13 +400,13 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_1_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 1; @@ -411,13 +416,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_1_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -429,13 +434,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_2_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 2; assert(oparg == CURRENT_OPARG()); @@ -443,13 +448,13 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_2_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 2; @@ -459,13 +464,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_2_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -477,13 +482,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_3_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 3; assert(oparg == CURRENT_OPARG()); @@ -491,13 +496,13 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_3_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 3; @@ -507,13 +512,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_3_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -525,13 +530,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_4_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 4; assert(oparg == CURRENT_OPARG()); @@ -539,13 +544,13 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_4_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 4; @@ -555,13 +560,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_4_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -573,13 +578,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_5_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 5; assert(oparg == CURRENT_OPARG()); @@ -587,13 +592,13 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_5_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 5; @@ -603,13 +608,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_5_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -621,13 +626,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_6_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 6; assert(oparg == CURRENT_OPARG()); @@ -635,13 +640,13 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_6_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 6; @@ -651,13 +656,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_6_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -669,13 +674,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_7_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 7; assert(oparg == CURRENT_OPARG()); @@ -683,13 +688,13 @@ value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_7_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 7; @@ -699,13 +704,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_7_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -717,26 +722,26 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = CURRENT_OPARG(); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -745,13 +750,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -762,13 +767,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_0_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); @@ -776,13 +781,13 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_0_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 0; @@ -792,13 +797,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_0_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -810,13 +815,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_1_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); @@ -824,13 +829,13 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_1_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 1; @@ -840,13 +845,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_1_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -858,13 +863,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_2_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 2; assert(oparg == CURRENT_OPARG()); @@ -872,13 +877,13 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_2_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 2; @@ -888,13 +893,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_2_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -906,13 +911,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_3_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 3; assert(oparg == CURRENT_OPARG()); @@ -920,13 +925,13 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_3_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 3; @@ -936,13 +941,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_3_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -954,13 +959,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_4_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 4; assert(oparg == CURRENT_OPARG()); @@ -968,13 +973,13 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_4_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 4; @@ -984,13 +989,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_4_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1002,13 +1007,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_5_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 5; assert(oparg == CURRENT_OPARG()); @@ -1016,13 +1021,13 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_5_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 5; @@ -1032,13 +1037,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_5_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1050,13 +1055,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_6_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 6; assert(oparg == CURRENT_OPARG()); @@ -1064,13 +1069,13 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_6_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 6; @@ -1080,13 +1085,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_6_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1098,13 +1103,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_7_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 7; assert(oparg == CURRENT_OPARG()); @@ -1112,13 +1117,13 @@ value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_7_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 7; @@ -1128,13 +1133,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_7_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1146,26 +1151,26 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = CURRENT_OPARG(); assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_Borrow(GETLOCAL(oparg)); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -1174,13 +1179,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_BORROW_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1191,26 +1196,26 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_AND_CLEAR_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = CURRENT_OPARG(); value = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_AND_CLEAR_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -1219,13 +1224,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FAST_AND_CLEAR_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1236,26 +1241,26 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = CURRENT_OPARG(); PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectBorrow(obj); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -1264,13 +1269,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1281,13 +1286,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_0_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 0; assert(oparg == CURRENT_OPARG()); @@ -1296,13 +1301,13 @@ value = PyStackRef_FromPyObjectBorrow(obj); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_0_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 0; @@ -1313,13 +1318,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_0_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1332,13 +1337,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_1_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 1; assert(oparg == CURRENT_OPARG()); @@ -1347,13 +1352,13 @@ value = PyStackRef_FromPyObjectBorrow(obj); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_1_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 1; @@ -1364,13 +1369,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_1_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1383,13 +1388,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_2_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 2; assert(oparg == CURRENT_OPARG()); @@ -1398,13 +1403,13 @@ value = PyStackRef_FromPyObjectBorrow(obj); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_2_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 2; @@ -1415,13 +1420,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_2_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1434,13 +1439,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_3_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = 3; assert(oparg == CURRENT_OPARG()); @@ -1449,13 +1454,13 @@ value = PyStackRef_FromPyObjectBorrow(obj); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_3_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 3; @@ -1466,13 +1471,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_3_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1485,13 +1490,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = CURRENT_OPARG(); assert(oparg < _PY_NSMALLPOSINTS); @@ -1499,13 +1504,13 @@ value = PyStackRef_FromPyObjectBorrow(obj); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -1515,13 +1520,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SMALL_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -1533,13 +1538,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_0_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = 0; @@ -1552,13 +1557,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_0_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1570,13 +1575,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_0_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1590,13 +1595,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_0_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1612,13 +1617,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_1_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = 1; @@ -1631,13 +1636,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_1_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1649,13 +1654,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_1_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1669,13 +1674,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_1_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1691,13 +1696,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_2_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = 2; @@ -1710,13 +1715,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_2_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1728,13 +1733,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_2_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1748,13 +1753,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_2_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1770,13 +1775,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_3_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = 3; @@ -1789,13 +1794,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_3_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1807,13 +1812,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_3_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1827,13 +1832,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_3_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1849,13 +1854,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_4_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = 4; @@ -1868,13 +1873,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_4_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1886,13 +1891,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_4_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1906,13 +1911,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_4_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1928,13 +1933,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_5_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = 5; @@ -1947,13 +1952,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_5_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1965,13 +1970,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_5_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -1985,13 +1990,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_5_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2007,13 +2012,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_6_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = 6; @@ -2026,13 +2031,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_6_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2044,13 +2049,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_6_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2064,13 +2069,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_6_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2086,13 +2091,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_7_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = 7; @@ -2105,13 +2110,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_7_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2123,13 +2128,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_7_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2143,13 +2148,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_7_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2165,13 +2170,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; oparg = CURRENT_OPARG(); @@ -2183,13 +2188,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2200,13 +2205,13 @@ trash = tmp; _tos_cache0 = trash; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2219,13 +2224,13 @@ _tos_cache1 = trash; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_FAST_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef trash; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2240,30 +2245,31 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_NOP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || @@ -2271,26 +2277,26 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_NOP_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; assert(PyStackRef_IsNull(value) || (!PyStackRef_RefcountOnObject(value)) || _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_NOP_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2299,13 +2305,13 @@ _Py_IsImmortal((PyStackRef_AsPyObjectBorrow(value)))); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_NOP_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2316,13 +2322,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_INT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); @@ -2330,26 +2336,26 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_INT_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; assert(PyLong_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_INT_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2358,13 +2364,13 @@ PyStackRef_CLOSE_SPECIALIZED(value, _PyLong_ExactDealloc); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_INT_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2375,13 +2381,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_FLOAT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); @@ -2389,26 +2395,26 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_FLOAT_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; assert(PyFloat_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_FLOAT_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2417,13 +2423,13 @@ PyStackRef_CLOSE_SPECIALIZED(value, _PyFloat_ExactDealloc); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_FLOAT_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2434,13 +2440,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_UNICODE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); @@ -2448,26 +2454,26 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_UNICODE_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(value))); PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_UNICODE_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2476,13 +2482,13 @@ PyStackRef_CLOSE_SPECIALIZED(value, _PyUnicode_ExactDealloc); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_UNICODE_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2493,56 +2499,57 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_TOP_OPARG_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; oparg = CURRENT_OPARG(); args = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_NULL_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; res = PyStackRef_NULL; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_NULL_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; res = PyStackRef_NULL; _tos_cache1 = res; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_NULL_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2551,30 +2558,31 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _END_FOR_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _POP_ITER_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef index_or_null; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2583,19 +2591,20 @@ iter = _stack_item_0; (void)index_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _END_SEND_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef index_or_null; _PyStackRef receiver; @@ -2612,21 +2621,22 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = val; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_NEGATIVE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -2636,8 +2646,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -2650,13 +2661,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_NEGATIVE_FLOAT_INPLACE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -2674,13 +2685,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_NEGATIVE_FLOAT_INPLACE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -2697,13 +2708,13 @@ _tos_cache1 = v; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_NEGATIVE_FLOAT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -2722,13 +2733,13 @@ _tos_cache1 = res; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_NOT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -2739,13 +2750,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_NOT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2755,13 +2766,13 @@ ? PyStackRef_True : PyStackRef_False; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_NOT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2773,13 +2784,13 @@ _tos_cache1 = res; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_NOT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2793,13 +2804,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -2808,13 +2819,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -2824,13 +2837,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_BOOL_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; if (!PyStackRef_BoolCheck(value)) { @@ -2843,13 +2856,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_BOOL_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; @@ -2862,13 +2875,13 @@ STAT_INC(TO_BOOL, hit); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_BOOL_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2884,13 +2897,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_BOOL_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -2909,13 +2922,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_INT_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -2929,13 +2942,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_INT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -2948,13 +2961,13 @@ _tos_cache1 = v; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -2969,13 +2982,13 @@ _tos_cache1 = res; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_LIST_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); @@ -2989,13 +3002,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_LIST_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; nos = stack_pointer[-1]; @@ -3011,13 +3024,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_LIST_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3033,13 +3046,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_LIST_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3058,13 +3071,13 @@ _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_LIST_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -3077,13 +3090,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_LIST_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -3096,13 +3109,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_LIST_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3118,13 +3131,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_LIST_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3143,13 +3156,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_SLICE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -3162,13 +3175,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_SLICE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -3181,13 +3194,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_SLICE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3203,13 +3216,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_SLICE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3228,13 +3241,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_LIST_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3249,13 +3262,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_LIST_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3269,13 +3282,13 @@ _tos_cache1 = v; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_LIST_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3291,13 +3304,13 @@ _tos_cache1 = res; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_NONE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -3312,13 +3325,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_NONE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -3333,13 +3346,13 @@ res = PyStackRef_False; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_NONE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -3357,13 +3370,13 @@ _tos_cache1 = res; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_NONE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -3384,13 +3397,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_COMPACT_ASCII_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); @@ -3409,13 +3422,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_COMPACT_ASCII_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; nos = stack_pointer[-1]; @@ -3437,13 +3450,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_COMPACT_ASCII_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3466,13 +3479,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_COMPACT_ASCII_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3499,13 +3512,13 @@ _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_UNICODE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); @@ -3519,13 +3532,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_UNICODE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; nos = stack_pointer[-1]; @@ -3541,13 +3554,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_UNICODE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3563,13 +3576,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_UNICODE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3588,13 +3601,13 @@ _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_UNICODE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); @@ -3607,13 +3620,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_UNICODE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; @@ -3626,13 +3639,13 @@ } _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_UNICODE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3648,13 +3661,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_UNICODE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3673,13 +3686,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_STR_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3693,13 +3706,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_STR_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3712,13 +3725,13 @@ _tos_cache1 = v; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TO_BOOL_STR_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3733,13 +3746,13 @@ _tos_cache1 = res; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _REPLACE_WITH_TRUE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3751,13 +3764,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _REPLACE_WITH_TRUE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3768,13 +3781,13 @@ _tos_cache1 = v; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _REPLACE_WITH_TRUE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3787,13 +3800,13 @@ _tos_cache1 = res; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNARY_INVERT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -3803,8 +3816,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -3817,13 +3831,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_INT_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -3837,13 +3851,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_INT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; left = stack_pointer[-1]; @@ -3859,13 +3873,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_INT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3881,13 +3895,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_INT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3906,13 +3920,13 @@ _tos_cache1 = left; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_INT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); @@ -3925,13 +3939,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_INT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; @@ -3944,13 +3958,13 @@ } _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_INT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3966,13 +3980,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_INT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -3991,13 +4005,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_OVERFLOWED_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -4012,13 +4026,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_OVERFLOWED_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; left = stack_pointer[-1]; @@ -4035,13 +4049,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_OVERFLOWED_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -4058,13 +4072,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_OVERFLOWED_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -4084,13 +4098,13 @@ _tos_cache1 = left; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_OVERFLOWED_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); @@ -4104,13 +4118,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_OVERFLOWED_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; @@ -4124,13 +4138,13 @@ } _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_OVERFLOWED_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -4147,13 +4161,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_OVERFLOWED_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -4173,13 +4187,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4207,13 +4221,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4243,13 +4257,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4279,13 +4293,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4313,13 +4327,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4349,13 +4363,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4385,13 +4399,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4419,13 +4433,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4455,13 +4469,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4491,13 +4505,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4520,13 +4534,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4551,13 +4565,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4582,13 +4596,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4611,13 +4625,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4642,13 +4656,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4673,13 +4687,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4702,13 +4716,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4733,13 +4747,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4764,13 +4778,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4793,13 +4807,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4824,13 +4838,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4855,13 +4869,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4884,13 +4898,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4915,13 +4929,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4946,13 +4960,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -4975,13 +4989,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5006,13 +5020,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5037,13 +5051,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_FLOAT_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); @@ -5057,13 +5071,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_FLOAT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; left = stack_pointer[-1]; @@ -5079,13 +5093,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_FLOAT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -5101,13 +5115,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_FLOAT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -5126,13 +5140,13 @@ _tos_cache1 = left; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FLOAT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); @@ -5145,13 +5159,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FLOAT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; @@ -5164,13 +5178,13 @@ } _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FLOAT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -5186,13 +5200,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FLOAT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -5211,13 +5225,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5247,13 +5261,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5287,13 +5301,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5327,13 +5341,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5363,13 +5377,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5403,13 +5417,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5443,13 +5457,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5479,13 +5493,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5519,13 +5533,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5559,13 +5573,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5583,13 +5597,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5608,13 +5622,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5632,13 +5646,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5656,13 +5670,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5681,13 +5695,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5705,13 +5719,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5729,13 +5743,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5754,13 +5768,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5778,13 +5792,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5802,13 +5816,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5827,13 +5841,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5851,13 +5865,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5875,13 +5889,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5900,13 +5914,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5924,13 +5938,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5948,13 +5962,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5973,13 +5987,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -5997,13 +6011,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_TRUEDIV_FLOAT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6025,9 +6039,10 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_SetString(PyExc_ZeroDivisionError, "float division by zero"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -6048,13 +6063,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6076,13 +6091,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6108,13 +6123,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6140,13 +6155,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6168,13 +6183,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6200,13 +6215,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6232,13 +6247,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_UNICODE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6265,13 +6280,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_UNICODE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6302,13 +6317,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_ADD_UNICODE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6339,13 +6354,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_INPLACE_ADD_UNICODE_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6378,9 +6393,13 @@ PyObject *right_o = PyStackRef_AsPyObjectSteal(right); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyUnicode_Append(&temp, right_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); *target_local = PyStackRef_NULL; if (temp == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -6391,13 +6410,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_LHS_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; left = stack_pointer[-2]; PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); @@ -6415,13 +6434,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_LHS_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; left = stack_pointer[-1]; @@ -6441,13 +6460,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_LHS_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -6467,13 +6486,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = left; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_LHS_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -6496,13 +6515,13 @@ _tos_cache1 = left; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_RHS_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; right = stack_pointer[-1]; PyObject *descr = (PyObject *)CURRENT_OPERAND0_64(); @@ -6520,13 +6539,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_RHS_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef _stack_item_0 = _tos_cache0; right = _stack_item_0; @@ -6546,13 +6565,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_RHS_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -6572,13 +6591,13 @@ _tos_cache1 = right; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_RHS_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -6601,13 +6620,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_EXTEND_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef _stack_item_0 = _tos_cache0; @@ -6625,10 +6644,11 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int match = (d->guard != NULL) ? d->guard(left_o, right_o) : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!match) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = right; @@ -6644,13 +6664,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_EXTEND_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -6671,8 +6691,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -6689,13 +6710,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_SLICE_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef stop; _PyStackRef start; _PyStackRef container; @@ -6717,8 +6738,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyList_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyTuple_CheckExact(container_o)) { stack_pointer[0] = container; @@ -6727,8 +6749,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyUnicode_CheckExact(container_o)) { stack_pointer[0] = container; @@ -6737,8 +6760,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { PyObject *slice = PySlice_New(start_o, stop_o, NULL); @@ -6752,29 +6776,40 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_GetItem(container_o, slice); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; } stack_pointer += 3; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = stop; stop = PyStackRef_NULL; stack_pointer[-3] = container; stack_pointer[-2] = start; stack_pointer[-1] = stop; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = start; start = PyStackRef_NULL; stack_pointer[-2] = start; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6786,13 +6821,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_SLICE_r30: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef stop; _PyStackRef start; _PyStackRef container; @@ -6810,9 +6845,11 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_AsPyObjectSteal(stop), + Py_None); + _PyFrame_StackPointerInvalidate(frame); int err; if (slice == NULL) { err = 1; @@ -6821,21 +6858,29 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 2; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-4] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -6846,13 +6891,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_LIST_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef res; @@ -6876,8 +6921,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = sub_st; @@ -6910,13 +6956,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_LIST_SLICE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef res; @@ -6935,8 +6981,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); STAT_INC(BINARY_OP, hit); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -6951,13 +6998,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_STR_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef str_st; _PyStackRef res; @@ -6997,13 +7044,13 @@ _tos_cache1 = s; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_USTR_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef str_st; _PyStackRef res; @@ -7049,13 +7096,13 @@ _tos_cache1 = s; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_TUPLE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); @@ -7069,13 +7116,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_TUPLE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; nos = stack_pointer[-1]; @@ -7091,13 +7138,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_TUPLE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7113,13 +7160,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_TUPLE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7138,13 +7185,13 @@ _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_TUPLE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -7157,13 +7204,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_TUPLE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -7176,13 +7223,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_TUPLE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7198,13 +7245,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_TUPLE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7223,13 +7270,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef tuple_st; sub_st = stack_pointer[-1]; @@ -7254,13 +7301,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef tuple_st; _PyStackRef _stack_item_0 = _tos_cache0; @@ -7288,13 +7335,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef tuple_st; _PyStackRef _stack_item_0 = _tos_cache0; @@ -7323,13 +7370,13 @@ _tos_cache1 = sub_st; _tos_cache0 = tuple_st; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef tuple_st; _PyStackRef _stack_item_0 = _tos_cache0; @@ -7362,13 +7409,13 @@ _tos_cache1 = tuple_st; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_TUPLE_INT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef tuple_st; _PyStackRef res; @@ -7393,13 +7440,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_TUPLE_INT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef tuple_st; _PyStackRef res; @@ -7425,13 +7472,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_TUPLE_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef tuple_st; _PyStackRef res; @@ -7456,13 +7503,13 @@ _tos_cache1 = ts; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT_SUBSCRIPT_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); @@ -7481,13 +7528,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT_SUBSCRIPT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; nos = stack_pointer[-1]; @@ -7509,13 +7556,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT_SUBSCRIPT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7538,13 +7585,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT_SUBSCRIPT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7571,13 +7618,13 @@ _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); @@ -7597,13 +7644,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; nos = stack_pointer[-1]; @@ -7626,13 +7673,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7658,13 +7705,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7691,13 +7738,13 @@ _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_DICT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -7710,13 +7757,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_DICT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -7729,13 +7776,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_DICT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7751,13 +7798,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_DICT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7776,13 +7823,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_DICT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -7795,13 +7842,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_DICT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -7814,13 +7861,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_DICT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7836,13 +7883,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_DICT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7861,13 +7908,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FROZENDICT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -7880,13 +7927,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FROZENDICT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -7899,13 +7946,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FROZENDICT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7921,13 +7968,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FROZENDICT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -7946,13 +7993,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef dict_st; _PyStackRef res; @@ -7972,8 +8019,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_SubscriptKnownHash(dict, sub, (Py_hash_t)hash); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -7987,13 +8035,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_DICT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef dict_st; _PyStackRef res; @@ -8012,8 +8060,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_Subscript(dict, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8027,13 +8076,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_CHECK_FUNC_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef container; _PyStackRef getitem; _PyStackRef _stack_item_0 = _tos_cache0; @@ -8079,13 +8128,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = container; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_INIT_CALL_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; @@ -8103,13 +8152,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_INIT_CALL_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; @@ -8128,13 +8177,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_INIT_CALL_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; @@ -8154,13 +8203,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_SUBSCR_INIT_CALL_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef getitem; _PyStackRef sub; _PyStackRef container; @@ -8179,13 +8228,13 @@ new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LIST_APPEND_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef v; _PyStackRef list; _PyStackRef _stack_item_0 = _tos_cache0; @@ -8199,13 +8248,13 @@ JUMP_TO_ERROR(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_ADD_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef v; _PyStackRef set; _PyStackRef _stack_item_0 = _tos_cache0; @@ -8216,9 +8265,10 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -8231,13 +8281,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_SUBSCR_r30: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub; _PyStackRef container; _PyStackRef v; @@ -8253,20 +8303,30 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-3] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -8277,13 +8337,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_SUBSCR_LIST_INT_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub_st; _PyStackRef list_st; _PyStackRef value; @@ -8336,21 +8396,22 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache1 = ss; _tos_cache0 = ls; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_SUBSCR_DICT_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub; _PyStackRef dict_st; _PyStackRef value; @@ -8370,16 +8431,18 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -8390,13 +8453,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_SUBSCR_DICT_KNOWN_HASH_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub; _PyStackRef dict_st; _PyStackRef value; @@ -8417,17 +8480,19 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2_KnownHash((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value), (Py_hash_t)hash); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -8438,13 +8503,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _DELETE_SUBSCR_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef sub; _PyStackRef container; _PyStackRef _stack_item_0 = _tos_cache0; @@ -8456,17 +8521,24 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -8477,13 +8549,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_INTRINSIC_1_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef v; @@ -8495,8 +8567,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8509,13 +8582,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_INTRINSIC_2_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value1_st; _PyStackRef value2_st; _PyStackRef res; @@ -8534,8 +8607,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8549,13 +8623,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_HEAP_SAFE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; value = PyStackRef_MakeHeapSafe(value); @@ -8563,26 +8637,26 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_HEAP_SAFE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; value = PyStackRef_MakeHeapSafe(value); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_HEAP_SAFE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -8591,13 +8665,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_HEAP_SAFE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -8608,13 +8682,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RETURN_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef retval; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -8622,12 +8696,15 @@ assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); _PyStackRef temp = retval; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -8635,13 +8712,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GET_AITER_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef obj; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -8658,16 +8735,18 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -8675,26 +8754,32 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -8703,13 +8788,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GET_ANEXT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef aiter; _PyStackRef awaitable; _PyStackRef _stack_item_0 = _tos_cache0; @@ -8718,8 +8803,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (awaitable_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8731,13 +8817,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GET_AWAITABLE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -8747,13 +8833,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -8763,15 +8851,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - /* _SEND is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _SEND_GEN_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef v; _PyStackRef receiver; _PyStackRef gen_frame; @@ -8811,236 +8897,758 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = receiver; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _YIELD_VALUE_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef retval; - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - retval = _stack_item_0; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - _PyStackRef temp = retval; - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; - FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - value = temp; - LLTRACE_RESUME_FRAME(); - _tos_cache0 = value; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; + case _GUARD_TOS_IS_NONE_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef val; + val = stack_pointer[-1]; + if (!PyStackRef_IsNone(val)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = val; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _POP_EXCEPT_r10: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef exc_value; - _PyStackRef _stack_item_0 = _tos_cache0; - exc_value = _stack_item_0; - _PyErr_StackItem *exc_info = tstate->exc_info; - stack_pointer[0] = exc_value; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _LOAD_COMMON_CONSTANT_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - oparg = CURRENT_OPARG(); - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - _tos_cache0 = value; + case _GUARD_TOS_IS_NONE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef val; + _PyStackRef _stack_item_0 = _tos_cache0; + val = _stack_item_0; + if (!PyStackRef_IsNone(val)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = val; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = val; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _LOAD_COMMON_CONSTANT_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; + case _GUARD_TOS_IS_NONE_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef val; _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - _tos_cache1 = value; + _PyStackRef _stack_item_1 = _tos_cache1; + val = _stack_item_1; + if (!PyStackRef_IsNone(val)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = val; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = val; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _LOAD_COMMON_CONSTANT_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; + case _GUARD_TOS_IS_NONE_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef val; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - oparg = CURRENT_OPARG(); - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - _tos_cache2 = value; + _PyStackRef _stack_item_2 = _tos_cache2; + val = _stack_item_2; + if (!PyStackRef_IsNone(val)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = val; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = val; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _LOAD_BUILD_CLASS_r01: { + case _GUARD_NOS_NOT_NULL_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef bc; - int err; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + nos = stack_pointer[-2]; + if (PyStackRef_IsNull(nos)) { + UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + JUMP_TO_JUMP_TARGET(); } - bc = PyStackRef_FromPyObjectSteal(bc_o); - _tos_cache0 = bc; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _STORE_NAME_r10: { + case _GUARD_NOS_NOT_NULL_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef v; + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; - oparg = CURRENT_OPARG(); - v = _stack_item_0; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - if (PyDict_CheckExact(ns)) { - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + nos = stack_pointer[-1]; + if (PyStackRef_IsNull(nos)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); } + _tos_cache1 = _stack_item_0; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); - } - _tos_cache0 = PyStackRef_ZERO_BITS; - _tos_cache1 = PyStackRef_ZERO_BITS; - _tos_cache2 = PyStackRef_ZERO_BITS; - SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _DELETE_NAME_r00: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - oparg = CURRENT_OPARG(); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_ERROR(); + case _GUARD_NOS_NOT_NULL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + nos = _stack_item_0; + if (PyStackRef_IsNull(nos)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + _tos_cache1 = _stack_item_1; + _tos_cache0 = nos; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_NOT_NULL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef nos; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + nos = _stack_item_1; + if (PyStackRef_IsNull(nos)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = nos; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + /* _SEND_VIRTUAL is not a viable micro-op for tier 2 because it is replaced */ + + case _SEND_VIRTUAL_TIER_TWO_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef none; + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + none = stack_pointer[-1]; + null_or_index = stack_pointer[-2]; + iter = stack_pointer[-3]; + assert(PyStackRef_IsNone(none)); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + next = none; + if (true) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + } + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_VIRTUAL_TIER_TWO_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef none; + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + none = _stack_item_0; + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + assert(PyStackRef_IsNone(none)); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + stack_pointer[0] = none; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + next = none; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = stack_pointer[0]; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + } + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_VIRTUAL_TIER_TWO_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef none; + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + none = _stack_item_1; + null_or_index = _stack_item_0; + iter = stack_pointer[-1]; + assert(PyStackRef_IsNone(none)); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + stack_pointer[0] = null_or_index; + stack_pointer[1] = none; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + next = none; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = stack_pointer[1]; + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + } + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _SEND_VIRTUAL_TIER_TWO_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef none; + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + none = _stack_item_2; + null_or_index = _stack_item_1; + iter = _stack_item_0; + assert(PyStackRef_IsNone(none)); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyObjectIndexPair next_index = CALL_TP_ITERITEM_NO_ESCAPE(iter_o, index); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + stack_pointer[0] = iter; + stack_pointer[1] = null_or_index; + stack_pointer[2] = none; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + next = none; + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = stack_pointer[2]; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + } + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + _tos_cache2 = next; + _tos_cache1 = null_or_index; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_3OS_ASYNC_GEN_ASEND_r03: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyAsyncGenASend_CheckExact(iter_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_3OS_ASYNC_GEN_ASEND_r13: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyAsyncGenASend_CheckExact(iter_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_0; + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_3OS_ASYNC_GEN_ASEND_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = stack_pointer[-1]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyAsyncGenASend_CheckExact(iter_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_1; + _tos_cache1 = _stack_item_0; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_3OS_ASYNC_GEN_ASEND_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyAsyncGenASend_CheckExact(iter_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + /* _SEND_ASYNC_GEN is not a viable micro-op for tier 2 because it is replaced */ + + case _SEND_ASYNC_GEN_TIER_TWO_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef null_in; + _PyStackRef iter; + _PyStackRef asend; + _PyStackRef null_out; + _PyStackRef retval; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + v = _stack_item_2; + null_in = _stack_item_1; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyAsyncGenASend_CheckExact(iter_o)); + PyObject *val = PyStackRef_AsPyObjectBorrow(v); + PyObject *retval_o; + stack_pointer[0] = iter; + stack_pointer[1] = null_in; + stack_pointer[2] = v; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); + _PyFrame_StackPointerInvalidate(frame); + if (what == PYGEN_ERROR) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + asend = iter; + null_out = null_in; + retval = PyStackRef_FromPyObjectSteal(retval_o); + if (what == PYGEN_RETURN) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = stack_pointer[-1]; + _tos_cache1 = stack_pointer[-2]; + _tos_cache0 = stack_pointer[-3]; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = retval; + _tos_cache1 = null_out; + _tos_cache0 = asend; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _YIELD_VALUE_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef retval; + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + retval = _stack_item_0; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + _PyStackRef temp = retval; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + DTRACE_FUNCTION_RETURN(); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; + FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = temp; + LLTRACE_RESUME_FRAME(); + _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _POP_EXCEPT_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef exc_value; + _PyStackRef _stack_item_0 = _tos_cache0; + exc_value = _stack_item_0; + _PyErr_StackItem *exc_info = tstate->exc_info; + stack_pointer[0] = exc_value; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + _PyFrame_StackPointerInvalidate(frame); + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _LOAD_COMMON_CONSTANT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _LOAD_COMMON_CONSTANT_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _LOAD_COMMON_CONSTANT_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + oparg = CURRENT_OPARG(); + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _LOAD_BUILD_CLASS_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef bc; + int err; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); + _PyFrame_StackPointerInvalidate(frame); + if (err < 0) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (bc_o == NULL) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + _tos_cache0 = bc; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _STORE_NAME_r10: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef v; + _PyStackRef _stack_item_0 = _tos_cache0; + oparg = CURRENT_OPARG(); + v = _stack_item_0; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + if (PyDict_CheckExact(ns)) { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + } + else { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + if (err) { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _tos_cache0 = PyStackRef_ZERO_BITS; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; + SET_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _DELETE_NAME_r00: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + oparg = CURRENT_OPARG(); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + _PyFrame_StackPointerInvalidate(frame); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyObject_DelItem(ns, name); + _PyFrame_StackPointerInvalidate(frame); + if (err != 0) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -9048,13 +9656,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef *top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -9063,9 +9671,13 @@ top = &stack_pointer[oparg]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9076,13 +9688,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_TWO_TUPLE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef val1; _PyStackRef val0; @@ -9106,21 +9718,22 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache1 = val0; _tos_cache0 = val1; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef val1; _PyStackRef val0; @@ -9136,13 +9749,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef val1; _PyStackRef val0; @@ -9157,13 +9770,13 @@ _tos_cache1 = val0; _tos_cache0 = val1; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef val1; _PyStackRef val0; @@ -9180,13 +9793,13 @@ _tos_cache1 = val1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef val2; _PyStackRef val1; @@ -9205,13 +9818,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef val2; _PyStackRef val1; @@ -9229,13 +9842,13 @@ _tos_cache1 = val1; _tos_cache0 = val2; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_TUPLE_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef *values; _PyStackRef _stack_item_0 = _tos_cache0; @@ -9258,19 +9871,20 @@ stack_pointer += oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef *values; _PyStackRef _stack_item_0 = _tos_cache0; @@ -9291,13 +9905,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_SEQUENCE_LIST_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef *values; _PyStackRef _stack_item_0 = _tos_cache0; @@ -9330,19 +9944,20 @@ stack_pointer += oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _UNPACK_EX_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef seq; _PyStackRef *top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -9351,9 +9966,13 @@ top = &stack_pointer[1 + (oparg & 0xFF) + (oparg >> 8)]; PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9364,13 +9983,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_ATTR_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef v; _PyStackRef _stack_item_0 = _tos_cache0; @@ -9384,17 +10003,24 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = PyStackRef_NULL; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-2] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -9405,13 +10031,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _DELETE_ATTR_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -9421,13 +10047,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9436,13 +10064,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_GLOBAL_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef v; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -9452,13 +10080,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9467,27 +10097,29 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _DELETE_GLOBAL_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -9495,33 +10127,34 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_LOCALS_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } locals = PyStackRef_FromPyObjectNew(l); _tos_cache0 = locals; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_LOCALS_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef locals; _PyStackRef _stack_item_0 = _tos_cache0; PyObject *l = LOCALS(); @@ -9530,9 +10163,10 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -9540,13 +10174,13 @@ _tos_cache1 = locals; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_LOCALS_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef locals; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -9557,9 +10191,10 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -9568,7 +10203,7 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -9576,62 +10211,72 @@ case _LOAD_NAME_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef v; oparg = CURRENT_OPARG(); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (l_v == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } v = PyStackRef_FromPyObjectSteal(v_o); _tos_cache0 = v; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_GLOBAL_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *res; oparg = CURRENT_OPARG(); res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(*res)) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9642,13 +10287,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_NULL_CONDITIONAL_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *null; oparg = CURRENT_OPARG(); null = &stack_pointer[0]; @@ -9658,13 +10303,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += (oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_GLOBALS_VERSION_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); if (!PyDict_CheckExact(dict)) { @@ -9680,13 +10325,13 @@ } assert(keys->dk_kind == DICT_KEYS_UNICODE); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_GLOBALS_VERSION_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); PyDictObject *dict = (PyDictObject *)GLOBALS(); @@ -9706,13 +10351,13 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_GLOBALS_VERSION_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); @@ -9736,13 +10381,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_GLOBALS_VERSION_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -9770,13 +10415,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_GLOBAL_MODULE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); @@ -9795,7 +10440,7 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -9816,13 +10461,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_GLOBAL_BUILTINS_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; uint16_t version = (uint16_t)CURRENT_OPERAND0_16(); uint16_t index = (uint16_t)CURRENT_OPERAND1_16(); @@ -9840,7 +10485,7 @@ } assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -9861,41 +10506,43 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _DELETE_FAST_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); oparg = CURRENT_OPARG(); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_CELL_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); oparg = CURRENT_OPARG(); PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); @@ -9906,43 +10553,46 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _DELETE_DEREF_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); oparg = CURRENT_OPARG(); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_FROM_DICT_OR_DEREF_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef class_dict_st; _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; @@ -9958,8 +10608,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -9968,9 +10619,10 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -9978,33 +10630,36 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); value = PyStackRef_FromPyObjectSteal(value_o); _tos_cache0 = value; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_DEREF_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; oparg = CURRENT_OPARG(); PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10012,13 +10667,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_DEREF_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef v; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -10028,21 +10683,22 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_FREE_VARS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); oparg = CURRENT_OPARG(); PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); @@ -10055,13 +10711,13 @@ frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_FREE_VARS_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); PyCodeObject *co = _PyFrame_GetCode(frame); @@ -10076,13 +10732,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_FREE_VARS_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -10099,13 +10755,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_FREE_VARS_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -10124,20 +10780,21 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BUILD_STRING_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *pieces; _PyStackRef str; oparg = CURRENT_OPARG(); pieces = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (str_o == NULL) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10151,13 +10808,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BUILD_INTERPOLATION_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *format; _PyStackRef str; _PyStackRef value; @@ -10177,14 +10834,16 @@ format_o = &_Py_STR(empty); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer += -(oparg & 1); @@ -10192,13 +10851,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (interpolation_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10208,13 +10869,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BUILD_TEMPLATE_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef interpolations; _PyStackRef strings; _PyStackRef template; @@ -10229,18 +10890,21 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (template_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10250,13 +10914,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BUILD_TUPLE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *values; _PyStackRef tup; oparg = CURRENT_OPARG(); @@ -10271,20 +10935,21 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BUILD_LIST_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *values; _PyStackRef list; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (list_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10296,13 +10961,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LIST_EXTEND_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable_st; _PyStackRef list_st; _PyStackRef i; @@ -10316,21 +10981,24 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10343,13 +11011,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_UPDATE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef set; _PyStackRef i; @@ -10361,9 +11029,10 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -10375,29 +11044,31 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BUILD_SET_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *values; _PyStackRef set; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = values[_i]; values[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); @@ -10408,22 +11079,25 @@ _PyStackRef value = values[i]; values[i] = PyStackRef_NULL; if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } if (err) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10434,20 +11108,21 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BUILD_MAP_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *values; _PyStackRef map; oparg = CURRENT_OPARG(); values = &stack_pointer[-oparg*2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (map_o == NULL) { stack_pointer += -oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10461,63 +11136,71 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SETUP_ANNOTATIONS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (ann_dict == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _DICT_UPDATE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef update; _PyStackRef dict; _PyStackRef upd; @@ -10531,29 +11214,37 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (has_keys == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'%T' object is not a mapping", update_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_ChainExceptions1(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } SET_CURRENT_CACHED_VALUES(0); @@ -10566,13 +11257,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _DICT_MERGE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef update; _PyStackRef dict; _PyStackRef callable; @@ -10590,13 +11281,18 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -10607,13 +11303,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAP_ADD_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef key; _PyStackRef dict_st; @@ -10630,12 +11326,13 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -10648,13 +11345,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SUPER_ATTR_ATTR_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_st; _PyStackRef class_st; _PyStackRef global_super_st; @@ -10694,20 +11391,30 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { @@ -10719,13 +11426,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_TYPE_VERSION_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; nos = stack_pointer[-2]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); @@ -10746,13 +11453,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_TYPE_VERSION_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; nos = stack_pointer[-1]; @@ -10776,13 +11483,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_TYPE_VERSION_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -10807,13 +11514,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = nos; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_TYPE_VERSION_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef nos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -10842,13 +11549,13 @@ _tos_cache1 = nos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_LOAD_SUPER_ATTR_METHOD_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef class_st; _PyStackRef global_super_st; oparg = CURRENT_OPARG(); @@ -10873,13 +11580,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_LOAD_SUPER_ATTR_METHOD_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef class_st; _PyStackRef global_super_st; _PyStackRef _stack_item_0 = _tos_cache0; @@ -10907,13 +11614,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_LOAD_SUPER_ATTR_METHOD_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef class_st; _PyStackRef global_super_st; _PyStackRef _stack_item_0 = _tos_cache0; @@ -10944,13 +11651,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_LOAD_SUPER_ATTR_METHOD_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef class_st; _PyStackRef global_super_st; _PyStackRef _stack_item_0 = _tos_cache0; @@ -10982,13 +11689,13 @@ _tos_cache1 = class_st; _tos_cache0 = global_super_st; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SUPER_ATTR_METHOD_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_st; _PyStackRef class_st; _PyStackRef global_super_st; @@ -11016,9 +11723,10 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (attr_o == NULL) { SET_CURRENT_CACHED_VALUES(0); @@ -11030,23 +11738,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); self_or_null = PyStackRef_NULL; stack_pointer += 1; } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-1] = class_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -11054,13 +11767,13 @@ _tos_cache0 = attr; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef *self_or_null; @@ -11074,8 +11787,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { stack_pointer[-1] = attr; stack_pointer += (oparg&1); @@ -11089,14 +11803,16 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -11110,13 +11826,13 @@ stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); @@ -11131,13 +11847,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; @@ -11152,13 +11868,13 @@ } _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11176,13 +11892,13 @@ _tos_cache1 = owner; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11203,13 +11919,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION_LOCKED_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); @@ -11228,13 +11944,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION_LOCKED_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; @@ -11253,13 +11969,13 @@ } _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION_LOCKED_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11281,13 +11997,13 @@ _tos_cache1 = owner; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_VERSION_LOCKED_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11312,13 +12028,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *type = (PyObject *)CURRENT_OPERAND0_64(); @@ -11332,13 +12048,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; @@ -11352,13 +12068,13 @@ } _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11375,13 +12091,13 @@ _tos_cache1 = owner; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TYPE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11401,13 +12117,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_MANAGED_OBJECT_HAS_VALUES_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -11422,13 +12138,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_MANAGED_OBJECT_HAS_VALUES_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; @@ -11443,13 +12159,13 @@ } _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_MANAGED_OBJECT_HAS_VALUES_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11467,13 +12183,13 @@ _tos_cache1 = owner; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_MANAGED_OBJECT_HAS_VALUES_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11494,13 +12210,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_INSTANCE_VALUE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -11533,13 +12249,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_INSTANCE_VALUE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -11573,13 +12289,13 @@ _tos_cache1 = o; _tos_cache0 = attr; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_INSTANCE_VALUE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -11617,13 +12333,13 @@ _tos_cache1 = attr; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_MODULE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -11650,7 +12366,7 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_value); if (attr_o == NULL) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = owner; @@ -11676,13 +12392,13 @@ _tos_cache0 = attr; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_WITH_HINT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -11764,13 +12480,13 @@ _tos_cache0 = attr; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_SLOT_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -11801,13 +12517,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_SLOT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -11839,13 +12555,13 @@ _tos_cache1 = o; _tos_cache0 = attr; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_SLOT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef o; @@ -11881,13 +12597,13 @@ _tos_cache1 = attr; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_ATTR_CLASS_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; uint32_t type_version = (uint32_t)CURRENT_OPERAND0_32(); @@ -11907,13 +12623,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_ATTR_CLASS_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; @@ -11934,13 +12650,13 @@ } _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_ATTR_CLASS_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -11965,13 +12681,13 @@ _tos_cache1 = owner; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_ATTR_CLASS_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -12000,13 +12716,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_CLASS_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; @@ -12019,24 +12735,25 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = attr; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_PROPERTY_FRAME_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef new_frame; oparg = CURRENT_OPARG(); @@ -12065,13 +12782,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_PROPERTY_FRAME_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef new_frame; _PyStackRef _stack_item_0 = _tos_cache0; @@ -12101,13 +12818,13 @@ new_frame = PyStackRef_Wrap(pushed_frame); _tos_cache0 = new_frame; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_PROPERTY_FRAME_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef new_frame; _PyStackRef _stack_item_0 = _tos_cache0; @@ -12141,13 +12858,13 @@ _tos_cache1 = new_frame; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_PROPERTY_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef new_frame; _PyStackRef _stack_item_0 = _tos_cache0; @@ -12185,13 +12902,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef new_frame; _PyStackRef _stack_item_0 = _tos_cache0; @@ -12228,13 +12945,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_DORV_NO_DICT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -12253,13 +12970,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_DORV_NO_DICT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; @@ -12278,13 +12995,13 @@ } _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_DORV_NO_DICT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -12306,13 +13023,13 @@ _tos_cache1 = owner; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_DORV_NO_DICT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -12337,13 +13054,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_ATTR_INSTANCE_VALUE_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef value; _PyStackRef o; @@ -12369,21 +13086,22 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = o; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOCK_OBJECT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; value = stack_pointer[-1]; if (!LOCK_OBJECT(PyStackRef_AsPyObjectBorrow(value))) { @@ -12395,13 +13113,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOCK_OBJECT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; value = _stack_item_0; @@ -12413,13 +13131,13 @@ } _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOCK_OBJECT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -12434,13 +13152,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOCK_OBJECT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -12458,13 +13176,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_ATTR_WITH_HINT_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef value; _PyStackRef o; @@ -12531,8 +13249,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); @@ -12541,21 +13260,22 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = o; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _STORE_ATTR_SLOT_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef value; _PyStackRef o; @@ -12582,21 +13302,22 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = o; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COMPARE_OP_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -12613,16 +13334,23 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = right; right = PyStackRef_NULL; stack_pointer[-1] = right; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = left; left = PyStackRef_NULL; stack_pointer[-2] = left; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -12631,9 +13359,13 @@ } if (oparg & 16) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res_bool = PyObject_IsTrue(res_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_bool < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -12647,13 +13379,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COMPARE_OP_FLOAT_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -12677,13 +13409,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COMPARE_OP_FLOAT_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -12708,13 +13440,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COMPARE_OP_FLOAT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -12738,13 +13470,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COMPARE_OP_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -12772,13 +13504,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COMPARE_OP_STR_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef res; @@ -12804,13 +13536,13 @@ _tos_cache1 = l; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _IS_OP_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -12829,13 +13561,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _IS_OP_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -12855,13 +13587,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _IS_OP_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -12880,13 +13612,13 @@ _tos_cache1 = l; _tos_cache0 = b; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CONTAINS_OP_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -12904,8 +13636,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -12919,13 +13652,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_SET_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -12938,13 +13671,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_SET_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -12957,13 +13690,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_SET_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -12979,13 +13712,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_ANY_SET_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13004,13 +13737,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_SET_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -13023,13 +13756,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_SET_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -13042,13 +13775,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_SET_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13064,13 +13797,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_SET_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13089,13 +13822,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FROZENSET_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); @@ -13108,13 +13841,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FROZENSET_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; @@ -13127,13 +13860,13 @@ } _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FROZENSET_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13149,13 +13882,13 @@ _tos_cache1 = tos; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_TOS_FROZENSET_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef tos; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13174,13 +13907,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CONTAINS_OP_SET_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -13200,8 +13933,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -13215,13 +13949,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CONTAINS_OP_DICT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -13241,8 +13975,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -13256,13 +13991,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_EG_MATCH_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef match_type_st; _PyStackRef exc_value_st; _PyStackRef rest; @@ -13278,19 +14013,24 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); @@ -13298,18 +14038,25 @@ } PyObject *match_o = NULL; PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { @@ -13323,8 +14070,9 @@ } if (!Py_IsNone(match_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -13332,13 +14080,13 @@ _tos_cache0 = rest; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_EXC_MATCH_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -13354,20 +14102,23 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); b = res ? PyStackRef_True : PyStackRef_False; _tos_cache1 = b; _tos_cache0 = left; @@ -13375,13 +14126,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _IMPORT_NAME_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef fromlist; _PyStackRef level; _PyStackRef res; @@ -13398,12 +14149,13 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level), oparg & 0x01); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer[0] = level; @@ -13411,22 +14163,27 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = level; level = PyStackRef_NULL; stack_pointer[-2] = level; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -13438,13 +14195,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _IMPORT_FROM_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef from; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -13457,18 +14214,20 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportFrom( tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer[0] = from; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportFrom( tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (res_o == NULL) { stack_pointer[-1] = from; @@ -13482,7 +14241,7 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -13492,7 +14251,7 @@ case _IS_NONE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef b; _PyStackRef _stack_item_0 = _tos_cache0; @@ -13506,18 +14265,19 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; } _tos_cache0 = b; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -13525,7 +14285,7 @@ case _GET_LEN_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef obj; _PyStackRef len; _PyStackRef _stack_item_0 = _tos_cache0; @@ -13534,8 +14294,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -13552,13 +14313,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MATCH_CLASS_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef names; _PyStackRef type; _PyStackRef subject; @@ -13580,11 +14341,12 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -13606,13 +14368,13 @@ stack_pointer[-3] = attrs; stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MATCH_MAPPING_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -13623,13 +14385,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MATCH_MAPPING_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef subject; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -13639,13 +14401,13 @@ _tos_cache1 = res; _tos_cache0 = subject; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MATCH_MAPPING_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef subject; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -13657,13 +14419,13 @@ _tos_cache1 = subject; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MATCH_SEQUENCE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -13674,13 +14436,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MATCH_SEQUENCE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef subject; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -13690,13 +14452,13 @@ _tos_cache1 = res; _tos_cache0 = subject; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MATCH_SEQUENCE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef subject; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -13708,13 +14470,13 @@ _tos_cache1 = subject; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MATCH_KEYS_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef keys; _PyStackRef subject; _PyStackRef values_or_none; @@ -13727,9 +14489,10 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (values_or_none_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -13741,13 +14504,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GET_ITER_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; @@ -13758,8 +14521,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsError(result)) { stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -13773,13 +14537,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_ITERATOR_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; iterable = stack_pointer[-1]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)); @@ -13793,13 +14557,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_ITERATOR_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef _stack_item_0 = _tos_cache0; iterable = _stack_item_0; @@ -13813,13 +14577,13 @@ STAT_INC(GET_ITER, hit); _tos_cache0 = iterable; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_ITERATOR_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13836,13 +14600,13 @@ _tos_cache1 = iterable; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_ITERATOR_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13862,13 +14626,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_ITER_VIRTUAL_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; iterable = stack_pointer[-1]; PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(iterable)); @@ -13882,13 +14646,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_ITER_VIRTUAL_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef _stack_item_0 = _tos_cache0; iterable = _stack_item_0; @@ -13902,13 +14666,13 @@ STAT_INC(GET_ITER, hit); _tos_cache0 = iterable; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_ITER_VIRTUAL_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13925,13 +14689,13 @@ _tos_cache1 = iterable; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_ITER_VIRTUAL_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13951,37 +14715,37 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_TAGGED_ZERO_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef zero; zero = PyStackRef_TagInt(0); _tos_cache0 = zero; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_TAGGED_ZERO_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef zero; _PyStackRef _stack_item_0 = _tos_cache0; zero = PyStackRef_TagInt(0); _tos_cache1 = zero; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_TAGGED_ZERO_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef zero; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -13990,13 +14754,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GET_ITER_TRAD_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; @@ -14006,13 +14770,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -14023,7 +14789,7 @@ _tos_cache0 = iter; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -14031,7 +14797,7 @@ case _FOR_ITER_TIER_TWO_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; @@ -14044,8 +14810,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { stack_pointer[-1] = null_or_index; @@ -14069,13 +14836,164 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TYPE_ITER_r02: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + iter = stack_pointer[-2]; + PyObject *expected_type = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != (PyTypeObject *)expected_type) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = stack_pointer[-1]; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TYPE_ITER_r12: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + iter = stack_pointer[-1]; + PyObject *expected_type = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != (PyTypeObject *)expected_type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_0; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TYPE_ITER_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + PyObject *expected_type = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != (PyTypeObject *)expected_type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TYPE_ITER_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_1; + PyObject *expected_type = (PyObject *)CURRENT_OPERAND0_64(); + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != (PyTypeObject *)expected_type) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _ITER_NEXT_INLINE_r23: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef next; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + PyObject *iternext_fn = (PyObject *)CURRENT_OPERAND0_64(); + assert(sizeof(iternextfunc) == sizeof(uintptr_t)); + volatile iternextfunc iternext_v = (iternextfunc)iternext_fn; + stack_pointer[0] = iter; + stack_pointer[1] = _stack_item_1; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyObject *item = iternext_v(PyStackRef_AsPyObjectBorrow(iter)); + _PyFrame_StackPointerInvalidate(frame); + if (item == NULL) { + if (_PyErr_Occurred(tstate)) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyEval_MonitorRaise(tstate, frame, frame->instr_ptr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + } + else { + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_ERROR(); + } + } + if (true) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + JUMP_TO_JUMP_TARGET(); + } + } + STAT_INC(FOR_ITER, hit); + next = PyStackRef_FromPyObjectSteal(item); + _tos_cache2 = next; + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_ITER_VIRTUAL_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); @@ -14089,13 +15007,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_ITER_VIRTUAL_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; iter = stack_pointer[-1]; @@ -14106,59 +15024,140 @@ SET_CURRENT_CACHED_VALUES(1); JUMP_TO_JUMP_TARGET(); } - _tos_cache1 = _stack_item_0; - _tos_cache0 = iter; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + _tos_cache1 = _stack_item_0; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_ITER_VIRTUAL_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + iter = _stack_item_0; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o)->_tp_iteritem == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = _stack_item_1; + _tos_cache0 = iter; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_NOS_ITER_VIRTUAL_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef iter; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + iter = _stack_item_1; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o)->_tp_iteritem == NULL) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = _stack_item_2; + _tos_cache1 = iter; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_NOT_NULL_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null_or_index; + null_or_index = stack_pointer[-1]; + if (PyStackRef_IsNull(null_or_index)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_NOT_NULL_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null_or_index; + _PyStackRef _stack_item_0 = _tos_cache0; + null_or_index = _stack_item_0; + if (PyStackRef_IsNull(null_or_index)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = null_or_index; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _GUARD_NOS_ITER_VIRTUAL_r22: { + case _GUARD_TOS_NOT_NULL_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null_or_index; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; - iter = _stack_item_0; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o)->_tp_iteritem == NULL) { + null_or_index = _stack_item_1; + if (PyStackRef_IsNull(null_or_index)) { UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; + _tos_cache1 = null_or_index; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); JUMP_TO_JUMP_TARGET(); } - _tos_cache1 = _stack_item_1; - _tos_cache0 = iter; + _tos_cache1 = null_or_index; + _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } - case _GUARD_NOS_ITER_VIRTUAL_r33: { + case _GUARD_TOS_NOT_NULL_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef iter; + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef null_or_index; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; - iter = _stack_item_1; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o)->_tp_iteritem == NULL) { + null_or_index = _stack_item_2; + if (PyStackRef_IsNull(null_or_index)) { UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; + _tos_cache2 = null_or_index; + _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); JUMP_TO_JUMP_TARGET(); } - _tos_cache2 = _stack_item_2; - _tos_cache1 = iter; + _tos_cache2 = null_or_index; + _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -14166,7 +15165,7 @@ case _FOR_ITER_VIRTUAL_TIER_TWO_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; @@ -14181,8 +15180,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { @@ -14208,7 +15208,7 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -14216,7 +15216,7 @@ case _ITER_CHECK_LIST_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; null_or_index = stack_pointer[-1]; @@ -14240,13 +15240,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_LIST_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14273,13 +15273,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_LIST_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14307,13 +15307,13 @@ _tos_cache1 = null_or_index; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_LIST_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14345,7 +15345,7 @@ _tos_cache1 = iter; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -14353,7 +15353,7 @@ case _GUARD_NOT_EXHAUSTED_LIST_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; null_or_index = stack_pointer[-1]; @@ -14372,13 +15372,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_LIST_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14399,13 +15399,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_LIST_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14426,13 +15426,13 @@ _tos_cache1 = null_or_index; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_LIST_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14456,7 +15456,7 @@ _tos_cache1 = iter; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -14464,7 +15464,7 @@ case _ITER_NEXT_LIST_TIER_TWO_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; @@ -14483,8 +15483,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result <= 0) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = null_or_index; @@ -14506,13 +15507,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_TUPLE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; null_or_index = stack_pointer[-1]; @@ -14529,13 +15530,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_TUPLE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14554,13 +15555,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_TUPLE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14579,13 +15580,13 @@ _tos_cache1 = null_or_index; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_TUPLE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14607,7 +15608,7 @@ _tos_cache1 = iter; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -14615,7 +15616,7 @@ case _GUARD_NOT_EXHAUSTED_TUPLE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; null_or_index = stack_pointer[-1]; @@ -14632,13 +15633,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_TUPLE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14657,13 +15658,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_TUPLE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14682,13 +15683,13 @@ _tos_cache1 = null_or_index; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_TUPLE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; @@ -14710,13 +15711,13 @@ _tos_cache1 = iter; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_NEXT_TUPLE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; @@ -14734,13 +15735,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_NEXT_TUPLE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; @@ -14759,13 +15760,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_NEXT_TUPLE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; @@ -14783,13 +15784,13 @@ _tos_cache1 = null_or_index; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_RANGE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); @@ -14810,13 +15811,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_RANGE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; iter = stack_pointer[-1]; @@ -14840,13 +15841,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_RANGE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -14871,13 +15872,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_CHECK_RANGE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -14906,7 +15907,7 @@ _tos_cache1 = iter; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -14914,7 +15915,7 @@ case _GUARD_NOT_EXHAUSTED_RANGE_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; iter = stack_pointer[-2]; _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); @@ -14929,13 +15930,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_RANGE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; iter = stack_pointer[-1]; @@ -14952,13 +15953,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_RANGE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -14975,13 +15976,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOT_EXHAUSTED_RANGE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -15001,13 +16002,13 @@ _tos_cache1 = iter; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_NEXT_RANGE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef next; iter = stack_pointer[-2]; @@ -15032,13 +16033,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_NEXT_RANGE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef next; _PyStackRef _stack_item_0 = _tos_cache0; @@ -15067,13 +16068,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ITER_NEXT_RANGE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef next; _PyStackRef _stack_item_0 = _tos_cache0; @@ -15102,13 +16103,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FOR_ITER_GEN_FRAME_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef gen_frame; oparg = CURRENT_OPARG(); @@ -15138,13 +16139,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FOR_ITER_GEN_FRAME_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef gen_frame; _PyStackRef _stack_item_0 = _tos_cache0; @@ -15177,13 +16178,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FOR_ITER_GEN_FRAME_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef iter; _PyStackRef gen_frame; _PyStackRef _stack_item_0 = _tos_cache0; @@ -15217,13 +16218,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = iter; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _INSERT_NULL_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self; _PyStackRef *method_and_self; _PyStackRef _stack_item_0 = _tos_cache0; @@ -15234,33 +16235,36 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_SPECIAL_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *method_and_self; oparg = CURRENT_OPARG(); method_and_self = &stack_pointer[-2]; PyObject *name = _Py_SpecialMethods[oparg].name; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err <= 0) { if (err == 0) { PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) ? _Py_SpecialMethods[oparg].error_suggestion : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(!_PyErr_Occurred(tstate)); assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -15269,13 +16273,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _WITH_EXCEPT_START_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef val; _PyStackRef lasti; _PyStackRef exit_self; @@ -15309,13 +16313,15 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -15327,13 +16333,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_EXC_INFO_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -15353,13 +16359,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_EXC_INFO_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -15378,13 +16384,13 @@ _tos_cache1 = new_exc; _tos_cache0 = prev_exc; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_EXC_INFO_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -15405,13 +16411,13 @@ _tos_cache1 = prev_exc; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); @@ -15426,13 +16432,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; @@ -15447,13 +16453,13 @@ } _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -15471,13 +16477,13 @@ _tos_cache1 = owner; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -15498,110 +16504,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_KEYS_VERSION_r01: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - owner = stack_pointer[-1]; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_KEYS_VERSION_r11: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - owner = _stack_item_0; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache0 = owner; - SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_KEYS_VERSION_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - owner = _stack_item_1; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = owner; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_KEYS_VERSION_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef owner; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - owner = _stack_item_2; - uint32_t keys_version = (uint32_t)CURRENT_OPERAND0_32(); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = owner; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_WITH_VALUES_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15619,13 +16528,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_WITH_VALUES_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15642,13 +16551,13 @@ _tos_cache1 = self; _tos_cache0 = attr; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_WITH_VALUES_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15667,13 +16576,13 @@ _tos_cache1 = attr; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_NO_DICT_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15692,13 +16601,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_NO_DICT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15716,13 +16625,13 @@ _tos_cache1 = self; _tos_cache0 = attr; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_NO_DICT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15742,13 +16651,13 @@ _tos_cache1 = attr; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; @@ -15759,20 +16668,21 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); _tos_cache0 = attr; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef _stack_item_0 = _tos_cache0; @@ -15784,20 +16694,21 @@ STAT_INC(LOAD_ATTR, hit); assert(descr != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); _tos_cache0 = attr; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_ATTR_METHOD_LAZY_DICT_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; owner = stack_pointer[-1]; uint16_t dictoffset = (uint16_t)CURRENT_OPERAND0_16(); @@ -15812,13 +16723,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_ATTR_METHOD_LAZY_DICT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; owner = _stack_item_0; @@ -15833,13 +16744,13 @@ } _tos_cache0 = owner; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_ATTR_METHOD_LAZY_DICT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -15857,13 +16768,13 @@ _tos_cache1 = owner; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_ATTR_METHOD_LAZY_DICT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -15884,13 +16795,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_LAZY_DICT_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15908,13 +16819,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_LAZY_DICT_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15931,13 +16842,13 @@ _tos_cache1 = self; _tos_cache0 = attr; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_ATTR_METHOD_LAZY_DICT_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef owner; _PyStackRef attr; _PyStackRef self; @@ -15956,13 +16867,13 @@ _tos_cache1 = attr; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAYBE_EXPAND_METHOD_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -15978,8 +16889,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; @@ -15987,7 +16899,7 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -15997,7 +16909,7 @@ case _PY_FRAME_GENERAL_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -16016,11 +16928,12 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -16032,13 +16945,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -16056,13 +16969,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION_INLINE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); assert(PyFunction_Check(callable_o)); @@ -16073,13 +16986,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION_INLINE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *callable_o = (PyObject *)CURRENT_OPERAND1_64(); @@ -16093,13 +17006,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION_INLINE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; uint32_t func_version = (uint32_t)CURRENT_OPERAND0_32(); @@ -16116,13 +17029,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION_INLINE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -16142,13 +17055,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_METHOD_VERSION_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -16178,13 +17091,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _EXPAND_METHOD_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -16200,19 +17113,20 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_NOT_PY_CALLABLE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -16228,13 +17142,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_NON_PY_GENERAL_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -16253,12 +17167,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -16272,13 +17187,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -16295,13 +17210,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -16316,32 +17231,33 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_PEP_523_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); if (IS_PEP523_HOOKED(tstate)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_PEP_523_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; if (IS_PEP523_HOOKED(tstate)) { UOP_STAT_INC(uopcode, miss); @@ -16351,13 +17267,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_PEP_523_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; if (IS_PEP523_HOOKED(tstate)) { @@ -16370,13 +17286,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_PEP_523_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -16392,13 +17308,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_EXACT_ARGS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -16414,13 +17330,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_STACK_SPACE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -16433,26 +17349,26 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_RECURSION_REMAINING_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_RECURSION_REMAINING_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; if (tstate->py_recursion_remaining <= 1) { UOP_STAT_INC(uopcode, miss); @@ -16462,13 +17378,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_RECURSION_REMAINING_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; if (tstate->py_recursion_remaining <= 1) { @@ -16481,13 +17397,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_RECURSION_REMAINING_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -16503,13 +17419,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _INIT_CALL_PY_EXACT_ARGS_0_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -16532,13 +17448,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _INIT_CALL_PY_EXACT_ARGS_1_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -16561,13 +17477,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _INIT_CALL_PY_EXACT_ARGS_2_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -16590,13 +17506,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _INIT_CALL_PY_EXACT_ARGS_3_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -16619,13 +17535,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _INIT_CALL_PY_EXACT_ARGS_4_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -16648,13 +17564,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _INIT_CALL_PY_EXACT_ARGS_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -16676,34 +17592,40 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PUSH_FRAME_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef new_frame; _PyStackRef _stack_item_0 = _tos_cache0; new_frame = _stack_item_0; assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_NULL_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; null = stack_pointer[-2]; if (!PyStackRef_IsNull(null)) { @@ -16716,13 +17638,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_NULL_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef _stack_item_0 = _tos_cache0; null = stack_pointer[-1]; @@ -16737,13 +17659,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_NULL_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -16758,13 +17680,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = null; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_NOS_NULL_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -16782,102 +17704,13 @@ _tos_cache1 = null; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NOT_NULL_r02: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = stack_pointer[-1]; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NOT_NULL_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - nos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_0; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NOT_NULL_r22: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - nos = _stack_item_0; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache1 = _stack_item_1; - _tos_cache0 = nos; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _GUARD_NOS_NOT_NULL_r33: { - CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef nos; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - _PyStackRef _stack_item_2 = _tos_cache2; - nos = _stack_item_1; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); - } - _tos_cache2 = _stack_item_2; - _tos_cache1 = nos; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_THIRD_NULL_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; null = stack_pointer[-3]; if (!PyStackRef_IsNull(null)) { @@ -16891,13 +17724,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_THIRD_NULL_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef _stack_item_0 = _tos_cache0; null = stack_pointer[-2]; @@ -16913,13 +17746,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_THIRD_NULL_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -16937,13 +17770,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_THIRD_NULL_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -16961,13 +17794,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = null; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TYPE_1_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -16982,13 +17815,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TYPE_1_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; callable = stack_pointer[-2]; @@ -17005,13 +17838,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TYPE_1_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -17030,13 +17863,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TYPE_1_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -17055,13 +17888,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_TYPE_1_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef res; _PyStackRef a; @@ -17077,13 +17910,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_TYPE_1_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef res; _PyStackRef a; @@ -17100,13 +17933,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_TYPE_1_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef res; _PyStackRef a; @@ -17124,13 +17957,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_TYPE_1_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef res; _PyStackRef a; @@ -17147,13 +17980,13 @@ _tos_cache1 = a; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_STR_1_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -17168,13 +18001,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_STR_1_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; callable = stack_pointer[-2]; @@ -17191,13 +18024,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_STR_1_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -17216,13 +18049,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_STR_1_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -17241,13 +18074,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_STR_1_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef res; _PyStackRef a; @@ -17265,8 +18098,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17279,13 +18113,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TUPLE_1_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -17300,13 +18134,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TUPLE_1_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; callable = stack_pointer[-2]; @@ -17323,13 +18157,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TUPLE_1_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -17348,13 +18182,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_TUPLE_1_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -17373,13 +18207,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_TUPLE_1_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef res; _PyStackRef a; @@ -17397,8 +18231,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17411,13 +18246,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_OBJECT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -17442,13 +18277,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _ALLOCATE_OBJECT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -17471,8 +18306,9 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (self_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17482,20 +18318,21 @@ callable = PyStackRef_FromPyObjectNew(init_func); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CREATE_INIT_FRAME_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self; _PyStackRef init; @@ -17505,22 +18342,25 @@ self = stack_pointer[-1 - oparg]; init = stack_pointer[-2 - oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -17531,13 +18371,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _EXIT_INIT_CHECK_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef should_be_none; _PyStackRef _stack_item_0 = _tos_cache0; should_be_none = _stack_item_0; @@ -17546,10 +18386,11 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } @@ -17557,13 +18398,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_BUILTIN_CLASS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -17580,13 +18421,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_BUILTIN_CLASS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -17602,11 +18443,12 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_CallBuiltinClass_StackRef( callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17614,20 +18456,21 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_BUILTIN_O_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; oparg = CURRENT_OPARG(); @@ -17654,13 +18497,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_BUILTIN_O_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -17679,8 +18522,9 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -17696,13 +18540,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_BUILTIN_FAST_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -17718,13 +18562,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_BUILTIN_FAST_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -17740,12 +18584,13 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFast_StackRef( callable, arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17753,20 +18598,21 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; oparg = CURRENT_OPARG(); callable = stack_pointer[-2 - oparg]; @@ -17782,13 +18628,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -17804,8 +18650,9 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17813,20 +18660,21 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LEN_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -17842,13 +18690,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LEN_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; callable = stack_pointer[-2]; @@ -17866,13 +18714,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LEN_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -17892,13 +18740,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LEN_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -17918,13 +18766,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_LEN_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef callable; _PyStackRef res; @@ -17943,8 +18791,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -17964,13 +18813,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_ISINSTANCE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; callable = stack_pointer[-4]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -17986,13 +18835,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_ISINSTANCE_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; callable = stack_pointer[-3]; @@ -18010,13 +18859,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_ISINSTANCE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -18036,13 +18885,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_ISINSTANCE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -18062,13 +18911,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_ISINSTANCE_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef cls; _PyStackRef instance; _PyStackRef null; @@ -18090,8 +18939,9 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (retval < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18100,31 +18950,34 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LIST_APPEND_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; callable = stack_pointer[-3]; PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); @@ -18140,13 +18993,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LIST_APPEND_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; callable = stack_pointer[-2]; @@ -18164,13 +19017,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LIST_APPEND_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -18190,13 +19043,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_LIST_APPEND_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -18216,13 +19069,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = callable; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_LIST_APPEND_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef self; _PyStackRef callable; @@ -18256,13 +19109,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_LIST_APPEND_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef self; _PyStackRef callable; @@ -18300,13 +19153,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_LIST_APPEND_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef self; _PyStackRef callable; @@ -18347,13 +19200,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_LIST_APPEND_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef arg; _PyStackRef self; _PyStackRef callable; @@ -18395,13 +19248,13 @@ _tos_cache1 = c; _tos_cache0 = none; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -18438,13 +19291,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_O_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -18467,8 +19320,9 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -18486,26 +19340,26 @@ stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_RECURSION_LIMIT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_RECURSION_LIMIT_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; if (_Py_ReachedRecursionLimit(tstate)) { UOP_STAT_INC(uopcode, miss); @@ -18515,13 +19369,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_RECURSION_LIMIT_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; if (_Py_ReachedRecursionLimit(tstate)) { @@ -18534,13 +19388,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_RECURSION_LIMIT_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -18556,13 +19410,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_O_INLINE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef callable; _PyStackRef res; @@ -18579,8 +19433,9 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(args[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(args[1]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -18598,13 +19453,13 @@ stack_pointer[-1 - oparg] = res; stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -18642,13 +19497,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -18668,7 +19523,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, @@ -18676,7 +19535,7 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18684,20 +19543,21 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_st; _PyStackRef callable; @@ -18709,7 +19569,11 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); volatile PyCFunctionFastWithKeywords cfunc_v = _PyCFunctionFastWithKeywords_CAST(cfunc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc_v, @@ -18717,7 +19581,7 @@ args - 1, oparg + 1 ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18725,20 +19589,21 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -18775,13 +19640,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_NOARGS_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -18803,8 +19668,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -18820,13 +19686,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef callable; _PyStackRef res; @@ -18842,8 +19708,9 @@ STAT_INC(CALL, hit); volatile PyCFunction cfunc_v = (PyCFunction)cfunc; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc_v, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -18859,13 +19726,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -18902,13 +19769,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_FAST_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_or_null; _PyStackRef callable; @@ -18928,7 +19795,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, @@ -18936,7 +19807,7 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18944,20 +19815,21 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef self_st; _PyStackRef callable; @@ -18970,7 +19842,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); volatile PyCFunctionFast cfunc_v = _PyCFunctionFast_CAST(cfunc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc_v, @@ -18978,7 +19854,7 @@ args - 1, oparg + 1 ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -18986,14 +19862,15 @@ _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = PyStackRef_ZERO_BITS; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -19001,7 +19878,7 @@ case _MAYBE_EXPAND_METHOD_KW_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; @@ -19021,8 +19898,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; } _tos_cache0 = _stack_item_0; @@ -19031,7 +19909,7 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -19039,7 +19917,7 @@ case _PY_FRAME_KW_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef kwnames; _PyStackRef *args; _PyStackRef self_or_null; @@ -19067,16 +19945,18 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -19088,13 +19968,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_FUNCTION_VERSION_KW_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -19116,13 +19996,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_METHOD_VERSION_KW_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef null; _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; @@ -19158,13 +20038,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _EXPAND_METHOD_KW_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef self_or_null; _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; @@ -19184,21 +20064,22 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _tos_cache0 = _stack_item_0; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_NOT_PY_CALLABLE_KW_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callable; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -19218,13 +20099,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_KW_NON_PY_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef kwnames; _PyStackRef *args; _PyStackRef self_or_null; @@ -19249,12 +20130,13 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -19268,13 +20150,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_CALLARGS_A_TUPLE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef callargs; _PyStackRef func; _PyStackRef _stack_item_0 = _tos_cache0; @@ -19290,15 +20172,17 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19306,16 +20190,17 @@ _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; } _tos_cache2 = _stack_item_2; _tos_cache1 = callargs; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -19323,7 +20208,7 @@ case _CHECK_IS_PY_CALLABLE_EX_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_st; func_st = stack_pointer[-4]; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); @@ -19343,13 +20228,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_PY_CALLABLE_EX_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_st; _PyStackRef _stack_item_0 = _tos_cache0; func_st = stack_pointer[-3]; @@ -19372,13 +20257,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_PY_CALLABLE_EX_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_st; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -19404,13 +20289,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_PY_CALLABLE_EX_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_st; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -19437,13 +20322,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _PY_FRAME_EX_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef kwargs_st; _PyStackRef callargs_st; _PyStackRef func_st; @@ -19468,10 +20353,11 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -19483,13 +20369,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_NOT_PY_CALLABLE_EX_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_st; func_st = stack_pointer[-4]; PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); @@ -19504,13 +20390,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_NOT_PY_CALLABLE_EX_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_st; _PyStackRef _stack_item_0 = _tos_cache0; func_st = stack_pointer[-3]; @@ -19527,13 +20413,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_NOT_PY_CALLABLE_EX_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_st; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -19552,13 +20438,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_IS_NOT_PY_CALLABLE_EX_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_st; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -19577,13 +20463,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CALL_FUNCTION_EX_NON_PY_GENERAL_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef kwargs_st; _PyStackRef callargs_st; _PyStackRef null; @@ -19608,23 +20494,27 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19634,13 +20524,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_FUNCTION_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef codeobj_st; _PyStackRef func; _PyStackRef co; @@ -19651,9 +20541,10 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (func_obj == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19668,13 +20559,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_FUNCTION_ATTRIBUTE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_in; _PyStackRef attr_st; _PyStackRef func_out; @@ -19694,13 +20585,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_FUNCTION_ATTRIBUTE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_in; _PyStackRef attr_st; _PyStackRef func_out; @@ -19721,13 +20612,13 @@ SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_FUNCTION_ATTRIBUTE_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_in; _PyStackRef attr_st; _PyStackRef func_out; @@ -19747,13 +20638,13 @@ *ptr = attr; _tos_cache0 = func_out; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_FUNCTION_ATTRIBUTE_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef func_in; _PyStackRef attr_st; _PyStackRef func_out; @@ -19775,25 +20666,27 @@ _tos_cache1 = func_out; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RETURN_GENERATOR_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } assert(STACK_LEVEL() <= 2); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); @@ -19802,23 +20695,25 @@ gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); _tos_cache0 = res; _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BUILD_SLICE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef *args; _PyStackRef slice; oparg = CURRENT_OPARG(); @@ -19828,13 +20723,14 @@ PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; args[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { @@ -19846,13 +20742,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CONVERT_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef result; _PyStackRef _stack_item_0 = _tos_cache0; @@ -19865,13 +20761,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19881,13 +20779,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FORMAT_SIMPLE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef res; _PyStackRef _stack_item_0 = _tos_cache0; @@ -19898,13 +20796,15 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -19918,13 +20818,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FORMAT_WITH_SPEC_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef fmt_spec; _PyStackRef value; _PyStackRef res; @@ -19937,16 +20837,23 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fmt_spec; fmt_spec = PyStackRef_NULL; stack_pointer[-1] = fmt_spec; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = value; value = PyStackRef_NULL; stack_pointer[-2] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -19958,13 +20865,13 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_1_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1]; @@ -19974,13 +20881,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_1_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -19989,13 +20896,13 @@ _tos_cache1 = top; _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_1_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20006,13 +20913,13 @@ _tos_cache1 = bottom; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_2_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-2]; @@ -20023,13 +20930,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_2_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20041,13 +20948,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_2_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20058,13 +20965,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_3_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-3]; @@ -20075,13 +20982,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_3_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20093,13 +21000,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_3_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20110,13 +21017,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_3_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20131,13 +21038,13 @@ stack_pointer[0] = bottom; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COPY_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef bottom; _PyStackRef top; oparg = CURRENT_OPARG(); @@ -20145,13 +21052,13 @@ top = PyStackRef_DUP(bottom); _tos_cache0 = top; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _BINARY_OP_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef rhs; _PyStackRef lhs; _PyStackRef res; @@ -20170,8 +21077,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); @@ -20185,13 +21093,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_2_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; top = stack_pointer[-1]; @@ -20204,13 +21112,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_2_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20224,13 +21132,13 @@ SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_2_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20243,13 +21151,13 @@ _tos_cache1 = top; _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_2_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20264,13 +21172,13 @@ _tos_cache1 = bottom; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_3_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; top = stack_pointer[-1]; @@ -20284,13 +21192,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_3_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20305,13 +21213,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_3_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20327,13 +21235,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_3_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20348,13 +21256,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SWAP_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef bottom; _PyStackRef _stack_item_0 = _tos_cache0; @@ -20367,7 +21275,7 @@ _tos_cache0 = top; SET_CURRENT_CACHED_VALUES(1); stack_pointer[-1 - (oparg-2)] = bottom; - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } @@ -20391,7 +21299,7 @@ case _GUARD_IS_TRUE_POP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; flag = stack_pointer[-1]; int is_true = PyStackRef_IsTrue(flag); @@ -20405,13 +21313,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_TRUE_POP_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; flag = _stack_item_0; @@ -20422,13 +21330,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_TRUE_POP_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20442,13 +21350,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_TRUE_POP_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20465,13 +21373,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_FALSE_POP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; flag = stack_pointer[-1]; int is_false = PyStackRef_IsFalse(flag); @@ -20485,13 +21393,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_FALSE_POP_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; flag = _stack_item_0; @@ -20502,13 +21410,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_FALSE_POP_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20522,13 +21430,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_FALSE_POP_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20545,13 +21453,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_4_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = 4; assert(oparg == CURRENT_OPARG()); @@ -20572,13 +21480,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_4_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 4; @@ -20596,13 +21504,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_4_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20623,13 +21531,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_4_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20653,13 +21561,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_5_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = 5; assert(oparg == CURRENT_OPARG()); @@ -20680,13 +21588,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_5_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 5; @@ -20704,13 +21612,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_5_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20731,13 +21639,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_5_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20761,13 +21669,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_6_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = 6; assert(oparg == CURRENT_OPARG()); @@ -20788,13 +21696,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_6_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 6; @@ -20812,13 +21720,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_6_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20839,13 +21747,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_6_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20869,13 +21777,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_7_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = 7; assert(oparg == CURRENT_OPARG()); @@ -20896,13 +21804,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_7_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 7; @@ -20920,13 +21828,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_7_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20947,13 +21855,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_7_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -20977,13 +21885,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = CURRENT_OPARG(); flag = stack_pointer[-1]; @@ -21003,13 +21911,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -21026,13 +21934,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21052,13 +21960,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_SET_POP_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21081,13 +21989,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_4_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = 4; assert(oparg == CURRENT_OPARG()); @@ -21108,13 +22016,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_4_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 4; @@ -21132,13 +22040,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_4_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21159,13 +22067,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_4_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21189,13 +22097,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_5_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = 5; assert(oparg == CURRENT_OPARG()); @@ -21216,13 +22124,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_5_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 5; @@ -21240,13 +22148,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_5_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21267,13 +22175,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_5_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21297,13 +22205,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_6_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = 6; assert(oparg == CURRENT_OPARG()); @@ -21324,13 +22232,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_6_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 6; @@ -21348,13 +22256,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_6_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21375,13 +22283,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_6_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21405,13 +22313,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_7_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = 7; assert(oparg == CURRENT_OPARG()); @@ -21432,13 +22340,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_7_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = 7; @@ -21456,13 +22364,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_7_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21483,13 +22391,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_7_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21513,13 +22421,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; oparg = CURRENT_OPARG(); flag = stack_pointer[-1]; @@ -21539,13 +22447,13 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); @@ -21562,13 +22470,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21588,13 +22496,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_BIT_IS_UNSET_POP_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef flag; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21617,13 +22525,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_NONE_POP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef val; val = stack_pointer[-1]; int is_none = PyStackRef_IsNone(val); @@ -21631,8 +22539,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (1) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -21642,21 +22551,22 @@ SET_CURRENT_CACHED_VALUES(0); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_NONE_POP_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef val; _PyStackRef _stack_item_0 = _tos_cache0; val = _stack_item_0; int is_none = PyStackRef_IsNone(val); if (!is_none) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (1) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -21664,13 +22574,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_NONE_POP_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef val; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21681,8 +22591,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (1) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = _stack_item_0; @@ -21694,13 +22605,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_NONE_POP_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef val; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -21713,8 +22624,9 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (1) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = _stack_item_1; @@ -21728,20 +22640,21 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IS_NOT_NONE_POP_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef val; _PyStackRef _stack_item_0 = _tos_cache0; val = _stack_item_0; int is_none = PyStackRef_IsNone(val); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(val); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (is_none) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -21751,44 +22664,44 @@ _tos_cache1 = PyStackRef_ZERO_BITS; _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _JUMP_TO_TOP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); JUMP_TO_JUMP_TARGET(); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_IP_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_IP_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_IP_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; PyObject *instr_ptr = (PyObject *)CURRENT_OPERAND0_64(); @@ -21796,13 +22709,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SET_IP_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -21812,13 +22725,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_STACK_SPACE_OPERAND_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); if (!_PyThreadState_HasStackSpace(tstate, framesize)) { @@ -21827,13 +22740,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_STACK_SPACE_OPERAND_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); assert(framesize <= INT_MAX); @@ -21845,13 +22758,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_STACK_SPACE_OPERAND_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; uint32_t framesize = (uint32_t)CURRENT_OPERAND0_32(); @@ -21866,13 +22779,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_STACK_SPACE_OPERAND_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -21890,13 +22803,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SAVE_RETURN_OFFSET_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); oparg = CURRENT_OPARG(); #if TIER_ONE frame->return_offset = (uint16_t)(next_instr - this_instr); @@ -21905,13 +22818,13 @@ frame->return_offset = oparg; #endif SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SAVE_RETURN_OFFSET_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; oparg = CURRENT_OPARG(); #if TIER_ONE @@ -21922,13 +22835,13 @@ #endif _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SAVE_RETURN_OFFSET_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; oparg = CURRENT_OPARG(); @@ -21941,13 +22854,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SAVE_RETURN_OFFSET_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -21962,13 +22875,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _EXIT_TRACE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) @@ -21978,13 +22891,20 @@ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("SIDE EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } #endif tstate->jit_exit = exit; @@ -21994,7 +22914,7 @@ case _EXIT_TRACE_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); _PyExitData *exit = (_PyExitData *)exit_p; @@ -22008,13 +22928,20 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("SIDE EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; } #endif @@ -22028,7 +22955,7 @@ case _EXIT_TRACE_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); @@ -22044,13 +22971,20 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("SIDE EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; } #endif @@ -22065,7 +22999,7 @@ case _EXIT_TRACE_r30: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22083,13 +23017,20 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("SIDE EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code], exit->is_control_flow); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; } #endif @@ -22105,7 +23046,7 @@ case _DYNAMIC_EXIT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; @@ -22113,13 +23054,20 @@ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("DYNAMIC EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } #endif SET_CURRENT_CACHED_VALUES(0); @@ -22128,7 +23076,7 @@ case _DYNAMIC_EXIT_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); #if defined(Py_DEBUG) && !defined(_Py_JIT) @@ -22140,13 +23088,20 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("DYNAMIC EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; } #endif @@ -22159,7 +23114,7 @@ case _DYNAMIC_EXIT_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; PyObject *exit_p = (PyObject *)CURRENT_OPERAND0_64(); @@ -22173,13 +23128,20 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("DYNAMIC EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; } #endif @@ -22193,7 +23155,7 @@ case _DYNAMIC_EXIT_r30: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22209,13 +23171,20 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); printf("DYNAMIC EXIT: [UOp "); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyUOpPrint(&next_uop[-1]); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); printf(", exit %tu, temp %d, target %d -> %s]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; } #endif @@ -22230,20 +23199,20 @@ case _CHECK_VALIDITY_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_VALIDITY_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); @@ -22253,13 +23222,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_VALIDITY_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; if (!current_executor->vm_data.valid) { @@ -22272,13 +23241,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _CHECK_VALIDITY_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22294,25 +23263,25 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_INLINE_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectNew(ptr); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_INLINE_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); @@ -22320,13 +23289,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_INLINE_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -22336,25 +23305,25 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_INLINE_BORROW_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); value = PyStackRef_FromPyObjectBorrow(ptr); _tos_cache0 = value; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_INLINE_BORROW_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); @@ -22362,13 +23331,13 @@ _tos_cache1 = value; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _LOAD_CONST_INLINE_BORROW_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef value; _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; @@ -22378,13 +23347,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RROT_3_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef middle; _PyStackRef bottom; @@ -22401,13 +23370,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RROT_3_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef middle; _PyStackRef bottom; @@ -22425,13 +23394,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RROT_3_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef middle; _PyStackRef bottom; @@ -22450,13 +23419,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _RROT_3_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef top; _PyStackRef middle; _PyStackRef bottom; @@ -22474,13 +23443,13 @@ _tos_cache1 = middle; _tos_cache0 = bottom; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _START_EXECUTOR_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); PyObject *executor = (PyObject *)CURRENT_OPERAND0_64(); #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); @@ -22491,8 +23460,9 @@ assert(tstate->jit_exit->executor == current_executor); assert(tstate->current_executor == executor); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyExecutor_ClearExit(tstate->jit_exit); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (true) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); @@ -22500,46 +23470,46 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_WARM_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); current_executor->vm_data.cold = false; SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_WARM_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; current_executor->vm_data.cold = false; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_WARM_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; current_executor->vm_data.cold = false; _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _MAKE_WARM_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22548,35 +23518,35 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FATAL_ERROR_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); assert(0); Py_FatalError("Fatal error uop executed."); SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FATAL_ERROR_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; assert(0); Py_FatalError("Fatal error uop executed."); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FATAL_ERROR_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; assert(0); @@ -22584,13 +23554,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _FATAL_ERROR_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22600,13 +23570,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _DEOPT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); @@ -22614,7 +23584,7 @@ case _DEOPT_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; stack_pointer[0] = _stack_item_0; stack_pointer += 1; @@ -22626,7 +23596,7 @@ case _DEOPT_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; stack_pointer[0] = _stack_item_0; @@ -22640,7 +23610,7 @@ case _DEOPT_r30: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22656,31 +23626,33 @@ case _HANDLE_PENDING_AND_DEOPT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } case _HANDLE_PENDING_AND_DEOPT_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } case _HANDLE_PENDING_AND_DEOPT_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; stack_pointer[0] = _stack_item_0; @@ -22688,15 +23660,16 @@ stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } case _HANDLE_PENDING_AND_DEOPT_r30: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22706,15 +23679,16 @@ stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_HandlePending(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } case _ERROR_POP_N_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0_32(); assert(oparg == 0); @@ -22725,68 +23699,68 @@ case _SPILL_OR_RELOAD_r01: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _tos_cache0 = stack_pointer[-1]; SET_CURRENT_CACHED_VALUES(1); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r02: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _tos_cache1 = stack_pointer[-1]; _tos_cache0 = stack_pointer[-2]; SET_CURRENT_CACHED_VALUES(2); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r03: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _tos_cache2 = stack_pointer[-1]; _tos_cache1 = stack_pointer[-2]; _tos_cache0 = stack_pointer[-3]; SET_CURRENT_CACHED_VALUES(3); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r10: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; SET_CURRENT_CACHED_VALUES(0); stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r12: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _tos_cache1 = _stack_item_0; _tos_cache0 = stack_pointer[-1]; SET_CURRENT_CACHED_VALUES(2); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r13: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _tos_cache2 = _stack_item_0; _tos_cache1 = stack_pointer[-1]; @@ -22794,13 +23768,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r20: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; SET_CURRENT_CACHED_VALUES(0); @@ -22808,13 +23782,13 @@ stack_pointer[1] = _stack_item_1; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r21: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _tos_cache0 = _stack_item_1; @@ -22822,13 +23796,13 @@ stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r23: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _tos_cache2 = _stack_item_1; @@ -22837,13 +23811,13 @@ SET_CURRENT_CACHED_VALUES(3); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r30: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22853,13 +23827,13 @@ stack_pointer[2] = _stack_item_2; stack_pointer += 3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r31: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22869,13 +23843,13 @@ stack_pointer[1] = _stack_item_1; stack_pointer += 2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _SPILL_OR_RELOAD_r32: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -22885,13 +23859,13 @@ stack_pointer[0] = _stack_item_0; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TIER2_RESUME_CHECK_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { UOP_STAT_INC(uopcode, miss); @@ -22908,13 +23882,13 @@ JUMP_TO_JUMP_TARGET(); } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TIER2_RESUME_CHECK_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -22935,13 +23909,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TIER2_RESUME_CHECK_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; #if defined(__EMSCRIPTEN__) @@ -22966,13 +23940,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _TIER2_RESUME_CHECK_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23001,13 +23975,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _COLD_EXIT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); @@ -23044,7 +24018,7 @@ case _COLD_DYNAMIC_EXIT_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _Py_CODEUNIT *target = frame->instr_ptr; SET_CURRENT_CACHED_VALUES(0); GOTO_TIER_ONE(target); @@ -23052,7 +24026,7 @@ case _GUARD_CODE_VERSION__PUSH_FRAME_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); assert(PyCode_Check(code)); @@ -23064,13 +24038,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION__PUSH_FRAME_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); @@ -23085,13 +24059,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION__PUSH_FRAME_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); @@ -23109,13 +24083,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION__PUSH_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23136,13 +24110,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_YIELD_VALUE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); assert(PyCode_Check(code)); @@ -23155,13 +24129,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_YIELD_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); @@ -23177,13 +24151,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_YIELD_VALUE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); @@ -23202,13 +24176,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_YIELD_VALUE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23230,13 +24204,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_RETURN_VALUE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); assert(PyCode_Check(code)); @@ -23249,13 +24223,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_RETURN_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); @@ -23271,13 +24245,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_RETURN_VALUE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); @@ -23296,13 +24270,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_RETURN_VALUE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23324,13 +24298,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_RETURN_GENERATOR_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); assert(PyCode_Check(code)); @@ -23343,13 +24317,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_RETURN_GENERATOR_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); PyObject *code = PyStackRef_AsPyObjectBorrow(frame->f_executable); @@ -23365,13 +24339,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_RETURN_GENERATOR_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; uint32_t version = (uint32_t)CURRENT_OPERAND0_32(); @@ -23390,13 +24364,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_CODE_VERSION_RETURN_GENERATOR_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23418,13 +24392,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP__PUSH_FRAME_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr; if (target != (_Py_CODEUNIT *)ip) { @@ -23435,13 +24409,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP__PUSH_FRAME_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr; @@ -23455,13 +24429,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP__PUSH_FRAME_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); @@ -23478,13 +24452,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP__PUSH_FRAME_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23504,13 +24478,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_YIELD_VALUE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; if (target != (_Py_CODEUNIT *)ip) { @@ -23522,13 +24496,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_YIELD_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND; @@ -23543,13 +24517,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_YIELD_VALUE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); @@ -23567,13 +24541,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_YIELD_VALUE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23594,13 +24568,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_RETURN_VALUE_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; if (target != (_Py_CODEUNIT *)ip) { @@ -23612,13 +24586,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_RETURN_VALUE_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; @@ -23633,13 +24607,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_RETURN_VALUE_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); @@ -23657,13 +24631,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_RETURN_VALUE_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23684,13 +24658,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_RETURN_GENERATOR_r00: { CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; if (target != (_Py_CODEUNIT *)ip) { @@ -23702,13 +24676,13 @@ } } SET_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_RETURN_GENERATOR_r11: { CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); _Py_CODEUNIT *target = frame->instr_ptr + frame->return_offset; @@ -23723,13 +24697,13 @@ } _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_RETURN_GENERATOR_r22: { CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; PyObject *ip = (PyObject *)CURRENT_OPERAND0_64(); @@ -23747,13 +24721,13 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } case _GUARD_IP_RETURN_GENERATOR_r33: { CHECK_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); _PyStackRef _stack_item_0 = _tos_cache0; _PyStackRef _stack_item_1 = _tos_cache1; _PyStackRef _stack_item_2 = _tos_cache2; @@ -23774,7 +24748,7 @@ _tos_cache1 = _stack_item_1; _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); break; } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 224426b7aa44bc2..f135f243c74e2f6 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -11,6 +11,7 @@ #include "pycore_opcode_utils.h" #include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() #include <stdbool.h> @@ -403,7 +404,6 @@ cfg_builder_maybe_start_new_block(cfg_builder *g) static bool cfg_builder_check(cfg_builder *g) { - assert(g->g_entryblock->b_iused > 0); for (basicblock *block = g->g_block_list; block != NULL; block = block->b_list) { assert(!_PyMem_IsPtrFreed(block)); if (block->b_instr != NULL) { @@ -1330,7 +1330,8 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) } if (opcode == LOAD_COMMON_CONSTANT) { assert(oparg < NUM_COMMON_CONSTANTS); - return Py_NewRef(_PyInterpreterState_GET()->common_consts[oparg]); + return PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg]); } if (constant == NULL) { @@ -1468,6 +1469,10 @@ maybe_instr_make_load_common_const(cfg_instr *instr, PyObject *newconst) && PyUnicode_GET_LENGTH(newconst) == 0) { oparg = CONSTANT_EMPTY_STR; } + else if (PyTuple_CheckExact(newconst) + && PyTuple_GET_SIZE(newconst) == 0) { + oparg = CONSTANT_EMPTY_TUPLE; + } else if (PyLong_CheckExact(newconst)) { int overflow; long val = PyLong_AsLongAndOverflow(newconst, &overflow); @@ -3273,6 +3278,7 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) index_map = PyMem_Malloc(nconsts * sizeof(Py_ssize_t)); if (index_map == NULL) { + PyErr_NoMemory(); goto end; } for (Py_ssize_t i = 1; i < nconsts; i++) { @@ -3325,6 +3331,7 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) /* adjust const indices in the bytecode */ reverse_index_map = PyMem_Malloc(nconsts * sizeof(Py_ssize_t)); if (reverse_index_map == NULL) { + PyErr_NoMemory(); goto end; } for (Py_ssize_t i = 0; i < nconsts; i++) { @@ -3754,6 +3761,7 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno) { assert(cfg_builder_check(g)); + assert(g->g_entryblock->b_iused > 0); /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ RETURN_IF_ERROR(translate_jump_labels_to_targets(g->g_entryblock)); diff --git a/Python/gc.c b/Python/gc.c index 134da107e1b61dc..201c621bcc3cb9b 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1405,13 +1405,13 @@ add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats) memcpy(cur_stats, prev_stats, sizeof(struct gc_generation_stats)); cur_stats->ts_start = stats->ts_start; - cur_stats->collections += 1; cur_stats->collected += stats->collected; cur_stats->uncollectable += stats->uncollectable; cur_stats->candidates += stats->candidates; cur_stats->duration += stats->duration; + cur_stats->heap_size = stats->heap_size; /* Publish ts_stop last so remote readers do not select a partially updated stats record as the latest collection. */ cur_stats->ts_stop = stats->ts_stop; @@ -1471,6 +1471,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) invoke_gc_callback(tstate, "start", generation, &stats); } + stats.heap_size = gcstate->heap_size; // ignore error: don't interrupt the GC if reading the clock fails (void)PyTime_PerfCounterRaw(&stats.ts_start); if (gcstate->debug & _PyGC_DEBUG_STATS) { @@ -1875,6 +1876,8 @@ _PyGC_Fini(PyInterpreterState *interp) GCState *gcstate = &interp->gc; Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); + PyMem_RawFree(gcstate->generation_stats); + gcstate->generation_stats = NULL; /* Prevent a subtle bug that affects sub-interpreters that use basic * single-phase init extensions (m_size == -1). Those extensions cause objects @@ -2097,6 +2100,8 @@ PyObject_GC_Del(void *op) PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); + GCState *gcstate = get_gc_state(); + gcstate->heap_size--; #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index b4fcd365592aa53..4e36189580bbf87 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -17,30 +17,6 @@ #include "pydtrace.h" -// Platform-specific includes for get_process_mem_usage(). -#ifdef _WIN32 - #include <windows.h> - #include <psapi.h> // For GetProcessMemoryInfo -#elif defined(__linux__) - #include <unistd.h> // For sysconf, getpid -#elif defined(__APPLE__) - #include <mach/mach.h> - #include <mach/task.h> // Required for TASK_VM_INFO - #include <unistd.h> // For sysconf, getpid -#elif defined(__FreeBSD__) - #include <sys/types.h> - #include <sys/sysctl.h> - #include <sys/user.h> // Requires sys/user.h for kinfo_proc definition - #include <kvm.h> - #include <unistd.h> // For sysconf, getpid - #include <fcntl.h> // For O_RDONLY - #include <limits.h> // For _POSIX2_LINE_MAX -#elif defined(__OpenBSD__) - #include <sys/types.h> - #include <sys/sysctl.h> - #include <sys/user.h> // For kinfo_proc - #include <unistd.h> // For sysconf, getpid -#endif // enable the "mark alive" pass of GC #define GC_ENABLE_MARK_ALIVE 1 @@ -2016,185 +1992,6 @@ cleanup_worklist(struct worklist *worklist) } } -// Return the memory usage (typically RSS + swap) of the process, in units of -// KB. Returns -1 if this operation is not supported or on failure. -static Py_ssize_t -get_process_mem_usage(void) -{ -#ifdef _WIN32 - // Windows implementation using GetProcessMemoryInfo - // Returns WorkingSetSize + PagefileUsage - PROCESS_MEMORY_COUNTERS pmc; - HANDLE hProcess = GetCurrentProcess(); - if (NULL == hProcess) { - // Should not happen for the current process - return -1; - } - - // GetProcessMemoryInfo returns non-zero on success - if (GetProcessMemoryInfo(hProcess, &pmc, sizeof(pmc))) { - // Values are in bytes, convert to KB. - return (Py_ssize_t)((pmc.WorkingSetSize + pmc.PagefileUsage) / 1024); - } - else { - return -1; - } - -#elif __linux__ - FILE* fp = fopen("/proc/self/status", "r"); - if (fp == NULL) { - return -1; - } - - char line_buffer[256]; - long long rss_kb = -1; - long long swap_kb = -1; - - while (fgets(line_buffer, sizeof(line_buffer), fp) != NULL) { - if (rss_kb == -1 && strncmp(line_buffer, "VmRSS:", 6) == 0) { - sscanf(line_buffer + 6, "%lld", &rss_kb); - } - else if (swap_kb == -1 && strncmp(line_buffer, "VmSwap:", 7) == 0) { - sscanf(line_buffer + 7, "%lld", &swap_kb); - } - if (rss_kb != -1 && swap_kb != -1) { - break; // Found both - } - } - fclose(fp); - - if (rss_kb != -1 && swap_kb != -1) { - return (Py_ssize_t)(rss_kb + swap_kb); - } - return -1; - -#elif defined(__APPLE__) - // --- MacOS (Darwin) --- - // Returns phys_footprint (RAM + compressed memory) - task_vm_info_data_t vm_info; - mach_msg_type_number_t count = TASK_VM_INFO_COUNT; - kern_return_t kerr; - - kerr = task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm_info, &count); - if (kerr != KERN_SUCCESS) { - return -1; - } - // phys_footprint is in bytes. Convert to KB. - return (Py_ssize_t)(vm_info.phys_footprint / 1024); - -#elif defined(__FreeBSD__) - // NOTE: Returns RSS only. Per-process swap usage isn't readily available - long page_size_kb = sysconf(_SC_PAGESIZE) / 1024; - if (page_size_kb <= 0) { - return -1; - } - - // Using /dev/null for vmcore avoids needing dump file. - // NULL for kernel file uses running kernel. - char errbuf[_POSIX2_LINE_MAX]; // For kvm error messages - kvm_t *kd = kvm_openfiles(NULL, "/dev/null", NULL, O_RDONLY, errbuf); - if (kd == NULL) { - return -1; - } - - // KERN_PROC_PID filters for the specific process ID - // n_procs will contain the number of processes returned (should be 1 or 0) - pid_t pid = getpid(); - int n_procs; - struct kinfo_proc *kp = kvm_getprocs(kd, KERN_PROC_PID, pid, &n_procs); - if (kp == NULL) { - kvm_close(kd); - return -1; - } - - Py_ssize_t rss_kb = -1; - if (n_procs > 0) { - // kp[0] contains the info for our process - // ki_rssize is in pages. Convert to KB. - rss_kb = (Py_ssize_t)kp->ki_rssize * page_size_kb; - } - else { - // Process with PID not found, shouldn't happen for self. - rss_kb = -1; - } - - kvm_close(kd); - return rss_kb; - -#elif defined(__OpenBSD__) - // NOTE: Returns RSS only. Per-process swap usage isn't readily available - long page_size_kb = sysconf(_SC_PAGESIZE) / 1024; - if (page_size_kb <= 0) { - return -1; - } - - struct kinfo_proc kp; - pid_t pid = getpid(); - int mib[6]; - size_t len = sizeof(kp); - - mib[0] = CTL_KERN; - mib[1] = KERN_PROC; - mib[2] = KERN_PROC_PID; - mib[3] = pid; - mib[4] = sizeof(struct kinfo_proc); // size of the structure we want - mib[5] = 1; // want 1 structure back - if (sysctl(mib, 6, &kp, &len, NULL, 0) == -1) { - return -1; - } - - if (len > 0) { - // p_vm_rssize is in pages on OpenBSD. Convert to KB. - return (Py_ssize_t)kp.p_vm_rssize * page_size_kb; - } - else { - // Process info not returned - return -1; - } -#else - // Unsupported platform - return -1; -#endif -} - -static bool -gc_should_collect_mem_usage(GCState *gcstate) -{ - Py_ssize_t mem = get_process_mem_usage(); - if (mem < 0) { - // Reading process memory usage is not support or failed. - return true; - } - int threshold = gcstate->young.threshold; - Py_ssize_t deferred = _Py_atomic_load_ssize_relaxed(&gcstate->deferred_count); - if (deferred > threshold * 40) { - // Too many new container objects since last GC, even though memory use - // might not have increased much. This is intended to avoid resource - // exhaustion if some objects consume resources but don't result in a - // memory usage increase. We use 40x as the factor here because older - // versions of Python would do full collections after roughly every - // 70,000 new container objects. - return true; - } - Py_ssize_t last_mem = _Py_atomic_load_ssize_relaxed(&gcstate->last_mem); - Py_ssize_t mem_threshold = Py_MAX(last_mem / 10, 128); - if ((mem - last_mem) > mem_threshold) { - // The process memory usage has increased too much, do a collection. - return true; - } - else { - // The memory usage has not increased enough, defer the collection and - // clear the young object count so we don't check memory usage again - // on the next call to gc_should_collect(). - PyMutex_Lock(&gcstate->mutex); - int young_count = _Py_atomic_exchange_int(&gcstate->young.count, 0); - _Py_atomic_store_ssize_relaxed(&gcstate->deferred_count, - gcstate->deferred_count + young_count); - PyMutex_Unlock(&gcstate->mutex); - return false; - } -} - static bool gc_should_collect(GCState *gcstate) { @@ -2214,7 +2011,7 @@ gc_should_collect(GCState *gcstate) // objects. return false; } - return gc_should_collect_mem_usage(gcstate); + return true; } static void @@ -2275,7 +2072,6 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, } state->gcstate->young.count = 0; - state->gcstate->deferred_count = 0; for (int i = 1; i <= generation; ++i) { state->gcstate->old[i-1].count = 0; } @@ -2379,11 +2175,6 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // to be freed. delete_garbage(state); - // Store the current memory usage, can be smaller now if breaking cycles - // freed some memory. - Py_ssize_t last_mem = get_process_mem_usage(); - _Py_atomic_store_ssize_relaxed(&state->gcstate->last_mem, last_mem); - // Append objects with legacy finalizers to the "gc.garbage" list. handle_legacy_finalizers(state); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bd2cf1c9cb19e5f..2cdf48c559a292c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -45,8 +45,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); @@ -62,8 +63,9 @@ PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); assert(_PyEval_BinaryOps[oparg]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -76,9 +78,10 @@ value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -86,8 +89,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -344,10 +348,11 @@ assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); assert(d != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int match = (d->guard != NULL) ? d->guard(left_o, right_o) : (Py_TYPE(left_o) == d->lhs_type && Py_TYPE(right_o) == d->rhs_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!match) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -363,9 +368,10 @@ assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = d->action(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -381,9 +387,10 @@ value = r; stack_pointer[-2] = res; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -391,8 +398,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -463,9 +471,13 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyUnicode_Append(&temp, right_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_DECREF_SPECIALIZED(right_o, _PyUnicode_ExactDealloc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); *target_local = PyStackRef_NULL; if (temp == NULL) { JUMP_TO_LABEL(error); @@ -670,8 +682,9 @@ assert(Py_TYPE(dict)->tp_as_mapping->mp_subscript == _PyDict_Subscript); STAT_INC(BINARY_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyDict_Subscript(dict, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -684,9 +697,10 @@ value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ds; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -694,8 +708,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -773,12 +788,18 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -837,8 +858,9 @@ } #ifdef Py_GIL_DISABLED _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); @@ -872,8 +894,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -927,8 +950,9 @@ assert(PySlice_Check(sub)); assert(PyList_CheckExact(list)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); STAT_INC(BINARY_OP, hit); if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -942,9 +966,10 @@ value = ss; stack_pointer[-2] = res; stack_pointer[-1] = ls; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -952,8 +977,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -1133,8 +1159,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -1405,18 +1432,21 @@ PyObject *res_o; if (PyList_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyList_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyTuple_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyTuple_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else if (PyUnicode_CheckExact(container_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyUnicode_BinarySlice(container_o, start_o, stop_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { PyObject *slice = PySlice_New(start_o, stop_o, NULL); @@ -1425,25 +1455,36 @@ } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_GetItem(container_o, slice); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = stop; stop = PyStackRef_NULL; stack_pointer[-1] = stop; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = start; start = PyStackRef_NULL; stack_pointer[-2] = start; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -1483,14 +1524,16 @@ format_o = &_Py_STR(empty); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (oparg & 1) { stack_pointer += -(oparg & 1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { stack_pointer += -(oparg & 1); @@ -1498,13 +1541,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (interpolation_o == NULL) { JUMP_TO_LABEL(error); } @@ -1527,8 +1572,9 @@ _PyStackRef list; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (list_o == NULL) { JUMP_TO_LABEL(error); } @@ -1551,8 +1597,9 @@ _PyStackRef map; values = &stack_pointer[-oparg*2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *map_o = _Py_BuildMap_StackRefSteal(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (map_o == NULL) { stack_pointer += -oparg*2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1577,17 +1624,19 @@ _PyStackRef set; values = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = values[_i]; values[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -1597,22 +1646,25 @@ _PyStackRef value = values[i]; values[i] = PyStackRef_NULL; if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } if (err) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectStealMortal(set_o); @@ -1638,13 +1690,14 @@ PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp; for (int _i = oparg; --_i >= 0;) { tmp = args[_i]; args[_i] = PyStackRef_NULL; PyStackRef_CLOSE(tmp); } - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (slice_o == NULL) { @@ -1669,8 +1722,9 @@ _PyStackRef str; pieces = &stack_pointer[-oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *str_o = _Py_BuildString_StackRefSteal(pieces, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (str_o == NULL) { stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1699,18 +1753,21 @@ PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (template_o == NULL) { JUMP_TO_LABEL(error); } @@ -1782,8 +1839,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Call(callable, self_or_null, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -1803,8 +1861,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL @@ -1826,11 +1885,12 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -1842,6 +1902,7 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -1851,7 +1912,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -1865,8 +1926,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -1951,8 +2013,9 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (self_o == NULL) { JUMP_TO_LABEL(error); } @@ -1961,32 +2024,36 @@ callable = PyStackRef_FromPyObjectNew(init_func); stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CREATE_INIT_FRAME { args = &stack_pointer[-oparg]; self = self_or_null; init = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; @@ -1999,12 +2066,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2062,8 +2135,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_FUNCTION_VERSION @@ -2142,12 +2216,18 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2219,8 +2299,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _CHECK_RECURSION_REMAINING @@ -2243,12 +2324,13 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -2270,12 +2352,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2326,27 +2414,30 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_CallBuiltinClass_StackRef( callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2354,14 +2445,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2413,28 +2506,31 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFast_StackRef( callable, arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2442,14 +2538,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2501,24 +2599,27 @@ } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_BuiltinCallFastWithKeywords_StackRef(callable, arguments, total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { args = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2526,14 +2627,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2605,8 +2708,9 @@ PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); _PyStackRef arg = args[0]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -2624,8 +2728,9 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -2633,14 +2738,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2684,23 +2791,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _CALL_FUNCTION_EX_NON_PY_GENERAL @@ -2717,23 +2827,27 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -2745,8 +2859,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -2788,23 +2903,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _CHECK_IS_PY_CALLABLE_EX @@ -2839,10 +2957,11 @@ stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -2865,12 +2984,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -2904,8 +3029,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallFunctionEx(func, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_FUNCTION_EX); @@ -2918,23 +3044,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX @@ -2957,34 +3086,39 @@ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } @@ -3003,10 +3137,11 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -3022,24 +3157,28 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -3051,8 +3190,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -3076,8 +3216,9 @@ value = stack_pointer[-1]; assert(oparg <= MAX_INTRINSIC_1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3088,9 +3229,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3117,8 +3259,9 @@ PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -3131,9 +3274,10 @@ value = vs2; stack_pointer[-2] = res; stack_pointer[-1] = vs1; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3141,8 +3285,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3193,8 +3338,9 @@ PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (retval < 0) { JUMP_TO_LABEL(error); } @@ -3202,18 +3348,21 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = retval ? PyStackRef_True : PyStackRef_False; assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); } @@ -3250,8 +3399,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -3271,8 +3421,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_KW @@ -3297,16 +3448,18 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -3317,6 +3470,7 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -3326,7 +3480,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3407,8 +3561,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // flush // _PY_FRAME_KW @@ -3427,17 +3582,19 @@ assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -3459,12 +3616,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -3519,12 +3682,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -3538,8 +3702,9 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -3615,16 +3780,18 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -3646,12 +3813,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -3703,8 +3876,9 @@ STAT_INC(CALL, hit); PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } @@ -3725,8 +3899,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3734,8 +3909,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3775,8 +3951,7 @@ // _GUARD_NOS_NOT_NULL { nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { + if (PyStackRef_IsNull(nos)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); JUMP_TO_PREDICTED(CALL); @@ -3820,8 +3995,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3829,8 +4005,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -3900,7 +4077,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFast_StackRef( callable, cfunc, @@ -3908,22 +4089,24 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -3931,14 +4114,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4012,7 +4197,11 @@ assert(self != NULL); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCFunctionFastWithKeywords cfunc = _PyCFunctionFastWithKeywords_CAST(method->d_method->ml_meth); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCallMethodDescriptorFastWithKeywords_StackRef( callable, cfunc, @@ -4020,22 +4209,24 @@ arguments, total_args ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callable; callable = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[-2 - oparg] = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_OPARG { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef_CloseStack(args, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4043,14 +4234,16 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4133,8 +4326,9 @@ STAT_INC(CALL, hit); PyCFunction cfunc = method->d_method->ml_meth; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4152,8 +4346,9 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4161,14 +4356,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4252,8 +4449,9 @@ PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); PyObject *arg = PyStackRef_AsPyObjectBorrow(arguments[1]); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); _Py_LeaveRecursiveCallTstate(tstate); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { @@ -4273,8 +4471,9 @@ stack_pointer += 1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4282,8 +4481,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -4291,14 +4491,16 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4353,12 +4555,13 @@ total_args++; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = _Py_VectorCall_StackRefSteal( callable, arguments, total_args, PyStackRef_NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -4372,8 +4575,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4483,12 +4687,18 @@ stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -4557,11 +4767,12 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable, locals, args, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (temp == NULL) { @@ -4583,12 +4794,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -4639,8 +4856,9 @@ assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4654,14 +4872,16 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4714,8 +4934,9 @@ assert(oparg == 1); STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -4729,14 +4950,16 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_AT_END { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -4798,8 +5021,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -4821,37 +5045,49 @@ PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, &match_o, &rest_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = match_type_st; match_type_st = PyStackRef_NULL; stack_pointer[-1] = match_type_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-2] = exc_value_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res < 0) { @@ -4863,8 +5099,9 @@ } if (!Py_IsNone(match_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } rest = PyStackRef_FromPyObjectSteal(rest_o); match = PyStackRef_FromPyObjectSteal(match_o); @@ -4892,19 +5129,22 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert(PyExceptionInstance_Check(left_o)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); b = res ? PyStackRef_True : PyStackRef_False; stack_pointer[0] = b; stack_pointer += 1; @@ -4939,37 +5179,55 @@ #endif assert(exc_value && PyExceptionInstance_Check(exc_value)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub_iter; sub_iter = value; stack_pointer[-4] = sub_iter; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = exc_value_st; exc_value_st = PyStackRef_NULL; stack_pointer[-1] = exc_value_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = last_sent_val; last_sent_val = PyStackRef_NULL; stack_pointer[-2] = last_sent_val; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = null_in; null_in = PyStackRef_NULL; stack_pointer[-3] = null_in; PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); null_out = null_in; none = PyStackRef_None; } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; @@ -5004,8 +5262,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); @@ -5018,16 +5277,23 @@ PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); assert((oparg >> 5) <= Py_GE); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = right; right = PyStackRef_NULL; stack_pointer[-1] = right; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = left; left = PyStackRef_NULL; stack_pointer[-2] = left; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -5035,9 +5301,13 @@ } if (oparg & 16) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res_bool = PyObject_IsTrue(res_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_bool < 0) { JUMP_TO_LABEL(error); } @@ -5294,8 +5564,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); @@ -5308,8 +5579,9 @@ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PySequence_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5322,9 +5594,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5332,8 +5605,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5376,8 +5650,9 @@ assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = PyDict_Contains(right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5390,9 +5665,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5400,8 +5676,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5444,8 +5721,9 @@ assert(PyAnySet_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PySet_Contains((PySetObject *)right_o, left_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res < 0) { JUMP_TO_LABEL(error); } @@ -5458,9 +5736,10 @@ value = r; stack_pointer[-2] = b; stack_pointer[-1] = l; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -5468,8 +5747,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5489,13 +5769,15 @@ assert(oparg >= FVC_STR && oparg <= FVC_ASCII); conv_fn = _PyEval_ConversionFuncs[oparg]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -5557,13 +5839,15 @@ owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -5582,13 +5866,15 @@ PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5603,18 +5889,20 @@ _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5628,16 +5916,18 @@ INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } DISPATCH(); @@ -5656,20 +5946,23 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } DISPATCH(); @@ -5688,17 +5981,24 @@ sub = stack_pointer[-1]; container = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -5730,13 +6030,18 @@ PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); PyObject *dupkey = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_MergeUniq(dict_o, update_o, &dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatKwargsError(tstate, callable_o, update_o, dupkey); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(dupkey); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } u = update; @@ -5747,8 +6052,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5772,29 +6078,37 @@ PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int has_keys = PyObject_HasAttrWithError(update_o, &_Py_ID(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (has_keys == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'%T' object is not a mapping", update_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_ChainExceptions1(exc); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } JUMP_TO_LABEL(error); @@ -5807,8 +6121,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -5832,27 +6147,39 @@ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } DISPATCH(); @@ -5870,8 +6197,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5896,8 +6224,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -5963,10 +6292,11 @@ should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyErr_Format(PyExc_TypeError, "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer += -1; @@ -6004,13 +6334,15 @@ PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); if (!PyUnicode_CheckExact(value_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -6040,16 +6372,23 @@ fmt_spec = stack_pointer[-1]; value = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fmt_spec; fmt_spec = PyStackRef_NULL; stack_pointer[-1] = fmt_spec; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = value; value = PyStackRef_NULL; stack_pointer[-2] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6086,8 +6425,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -6097,8 +6437,9 @@ // _FOR_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -6168,12 +6509,18 @@ assert(!IS_PEP523_HOOKED(tstate)); _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -6238,8 +6585,9 @@ _PyObject_GC_IS_SHARED(list_o)); STAT_INC(FOR_ITER, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); @@ -6397,15 +6745,14 @@ next_instr += 2; INSTRUCTION_STATS(FOR_ITER_VIRTUAL); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; _PyStackRef null_or_index; + _PyStackRef iter; _PyStackRef next; /* Skip 1 cache entry */ - // _GUARD_NOS_ITER_VIRTUAL + // _GUARD_TOS_NOT_NULL { - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o)->_tp_iteritem == NULL) { + null_or_index = stack_pointer[-1]; + if (PyStackRef_IsNull(null_or_index)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); @@ -6413,12 +6760,13 @@ } // _FOR_ITER_VIRTUAL { - null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); Py_ssize_t index = PyStackRef_UntagInt(null_or_index); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); PyObject *next_o = next_index.object; index = next_index.index; if (next_o == NULL) { @@ -6458,38 +6806,46 @@ } if (getter == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' requires an object with " "__aiter__ method, got %.100s", type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "'async for' received an object from __aiter__ " "that does not implement __anext__: %.100s", Py_TYPE(iter_o)->tp_name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); @@ -6511,8 +6867,9 @@ _PyStackRef awaitable; aiter = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (awaitable_o == NULL) { JUMP_TO_LABEL(error); } @@ -6535,13 +6892,15 @@ _PyStackRef iter; iterable = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (iter_o == NULL) { JUMP_TO_LABEL(error); } @@ -6575,8 +6934,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_GetIter(iterable, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(GET_ITER); @@ -6586,8 +6946,9 @@ // _GET_ITER { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef result = _PyEval_GetIter(iterable, &index_or_null, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsError(result)) { JUMP_TO_LABEL(pop_1_error); } @@ -6682,8 +7043,9 @@ _PyStackRef len; obj = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (len_i < 0) { JUMP_TO_LABEL(error); } @@ -6713,15 +7075,17 @@ PyObject *res_o; if (PyLazyImport_CheckExact(PyStackRef_AsPyObjectBorrow(from))) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportFrom( tstate, frame, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportFrom( tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (res_o == NULL) { JUMP_TO_LABEL(error); @@ -6750,31 +7114,37 @@ PyObject *res_o; if (!(oparg & 0x02)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_LazyImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level), oparg & 0x01); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = _PyEval_ImportName(tstate, BUILTINS(), GLOBALS(), LOCALS(), name, PyStackRef_AsPyObjectBorrow(fromlist), PyStackRef_AsPyObjectBorrow(level)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = fromlist; fromlist = PyStackRef_NULL; stack_pointer[-1] = fromlist; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = level; level = PyStackRef_NULL; stack_pointer[-2] = level; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (res_o == NULL) { @@ -6819,8 +7189,9 @@ stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL @@ -6843,11 +7214,12 @@ stack_pointer[-2 - oparg] = func; stack_pointer[-1 - oparg] = maybe_self; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg0 ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -6870,12 +7242,13 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, total_args, NULL, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -6884,7 +7257,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -6894,7 +7268,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -2 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -6908,8 +7282,9 @@ stack_pointer += -1 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -6943,23 +7318,26 @@ PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); if (!PyTuple_CheckExact(callargs_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (tuple_o == NULL) { JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _DO_CALL_FUNCTION_EX @@ -6982,34 +7360,39 @@ PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyFunction_Check(func) && !PyMethod_Check(func)) { if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } @@ -7028,10 +7411,11 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( tstate, func_st, locals, nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (new_frame == NULL) { @@ -7047,24 +7431,28 @@ assert(kwargs == NULL || PyDict_CheckExact(kwargs)); stack_pointer[-2] = callargs_st; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (result_o == NULL) { JUMP_TO_LABEL(error); } @@ -7076,8 +7464,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7117,8 +7506,9 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _MONITOR_CALL_KW @@ -7139,10 +7529,11 @@ stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7166,17 +7557,19 @@ { int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( tstate, callable, locals, arguments, positional_args, kwnames_o, frame ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (new_frame == NULL) { JUMP_TO_LABEL(error); } @@ -7184,7 +7577,8 @@ frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject* res_o = _Py_VectorCallInstrumentation_StackRefSteal( callable, arguments, @@ -7194,7 +7588,7 @@ frame, this_instr, tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { stack_pointer += -3 - oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -7234,27 +7628,39 @@ PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); assert(exc && PyExceptionInstance_Check(exc)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = exc_st; exc_st = PyStackRef_NULL; stack_pointer[-1] = exc_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = awaitable_st; awaitable_st = PyStackRef_NULL; stack_pointer[-2] = awaitable_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); } else { Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } } @@ -7276,8 +7682,9 @@ receiver = stack_pointer[-3]; if (PyStackRef_GenCheck(receiver)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7285,8 +7692,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7310,8 +7718,9 @@ PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7322,8 +7731,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7344,8 +7754,9 @@ null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -7375,9 +7786,10 @@ INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (next_opcode < 0) { JUMP_TO_LABEL(error); } @@ -7404,8 +7816,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7451,9 +7864,13 @@ next_instr = this_instr; } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); original_opcode = _Py_call_instrumentation_line( tstate, frame, this_instr, prev_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); if (original_opcode < 0) { next_instr = this_instr+1; JUMP_TO_LABEL(error); @@ -7468,6 +7885,7 @@ PAUSE_ADAPTIVE_COUNTER(cache->counter); } opcode = original_opcode; + PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); } @@ -7499,25 +7917,33 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -7527,45 +7953,56 @@ { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { @@ -7573,9 +8010,13 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -7630,8 +8071,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -7681,8 +8123,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } stack_pointer += -1; @@ -7709,8 +8152,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); } else { @@ -7760,9 +8204,10 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } @@ -7787,8 +8232,9 @@ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7801,8 +8247,9 @@ { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -7811,9 +8258,13 @@ // _MONITOR_RESUME { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation( tstate, oparg == 0 ? PY_MONITORING_EVENT_PY_START : PY_MONITORING_EVENT_PY_RESUME, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + // Explicit stack reload stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7842,10 +8293,11 @@ { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_RETURN, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -7863,12 +8315,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -7889,6 +8344,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; _PyStackRef val; _PyStackRef value; _PyStackRef retval; @@ -7896,14 +8352,21 @@ { val = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_arg( tstate, PY_MONITORING_EVENT_PY_YIELD, frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { + // Explicit stack reload + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackAssertInvalid(frame); next_instr = frame->instr_ptr; DISPATCH(); } @@ -7925,24 +8388,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -7975,12 +8442,14 @@ _PyStackRef executor = frame->localsplus[0]; assert(tstate->current_executor == NULL); if (!PyStackRef_IsNull(executor)) { + assert(PyStackRef_TYPE(executor) == &_PyUOpExecutor_Type); tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 1; } #endif @@ -8017,8 +8486,9 @@ stack_pointer[-2] = b; stack_pointer[-1] = l; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -8026,8 +8496,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -8058,8 +8529,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8087,8 +8559,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8155,8 +8628,9 @@ // _CHECK_PERIODIC { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -8222,21 +8696,24 @@ PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); if (matches && (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Clear(tstate); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, "Value after * must be an iterable, not %.200s", Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } @@ -8249,8 +8726,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -8279,8 +8757,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); @@ -8294,22 +8773,25 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(pop_1_error); } } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr = _PyObject_GetAttrStackRef(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer[-1] = attr; stack_pointer += (oparg&1); ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(attr)) { JUMP_TO_LABEL(error); } @@ -8362,11 +8844,12 @@ assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { @@ -8430,11 +8913,12 @@ assert(descr != NULL); attr = PyStackRef_FromPyObjectNew(descr); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = attr; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _PUSH_NULL_CONDITIONAL { @@ -8526,12 +9010,18 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -8608,8 +9098,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -8764,18 +9255,7 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } + /* Skip 2 cache entries */ // _LOAD_ATTR_METHOD_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); @@ -8832,7 +9312,7 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_value); if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); @@ -8858,8 +9338,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -8911,8 +9392,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; @@ -8958,18 +9440,7 @@ JUMP_TO_PREDICTED(LOAD_ATTR); } } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); - } - } + /* Skip 2 cache entries */ // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES { PyObject *descr = read_obj(&this_instr[6].cache); @@ -8979,8 +9450,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); attr = PyStackRef_FromPyObjectNew(descr); } stack_pointer[0] = attr; @@ -9062,12 +9534,18 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); @@ -9131,8 +9609,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9248,8 +9727,9 @@ value = o; stack_pointer[-1] = attr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } /* Skip 5 cache entries */ // _PUSH_NULL_CONDITIONAL @@ -9275,16 +9755,18 @@ _PyStackRef bc; int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *bc_o = _PyMapping_GetOptionalItem2(BUILTINS(), &_Py_ID(__build_class__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); @@ -9304,7 +9786,7 @@ INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + value = PyStackRef_DupImmortal(tstate->interp->common_consts[oparg]); stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); @@ -9339,15 +9821,17 @@ _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(value)) { stack_pointer[0] = value; stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } stack_pointer[0] = value; @@ -9440,11 +9924,12 @@ _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, UNBOUNDLOCAL_ERROR_MSG, PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); @@ -9493,8 +9978,9 @@ name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* value_o = _PyMapping_GetOptionalItem2(class_dict, name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -9502,17 +9988,19 @@ PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); value_o = PyCell_GetRef(cell); if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); value = PyStackRef_FromPyObjectSteal(value_o); stack_pointer[0] = value; stack_pointer += 1; @@ -9534,13 +10022,15 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyMapping_GetOptionalItem2(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(mod_or_class_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -9548,58 +10038,71 @@ if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), (PyDictObject *)BUILTINS(), name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(GLOBALS(), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); v_o = _PyMapping_GetOptionalItem2(BUILTINS(), name, &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_FormatExcCheckArg( tstate, PyExc_NameError, NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } @@ -9635,8 +10138,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); @@ -9651,8 +10155,9 @@ res = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (PyStackRef_IsNull(*res)) { JUMP_TO_LABEL(error); } @@ -9718,7 +10223,7 @@ } assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); @@ -9785,7 +10290,7 @@ assert(keys->dk_kind == DICT_KEYS_UNICODE); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_CONSUME(entries[index].me_value); if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); @@ -9828,9 +10333,10 @@ PyObject *l = LOCALS(); if (l == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); @@ -9851,34 +10357,43 @@ _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (v_o == NULL) { JUMP_TO_LABEL(error); } if (PyLazyImport_CheckExact(v_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyObject *l_v = _PyImport_LoadLazyImportTstate(tstate, v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (l_v == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(v_o); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(v_o, l_v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -9929,21 +10444,24 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err <= 0) { if (err == 0) { PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) ? _Py_SpecialMethods[oparg].error_suggestion : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(!_PyErr_Occurred(tstate)); assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } JUMP_TO_LABEL(error); } @@ -9979,8 +10497,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -9996,25 +10515,33 @@ if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_CALL, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); JUMP_TO_LABEL(error); @@ -10024,45 +10551,56 @@ { PyObject *stack[] = {class, self}; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _Py_call_instrumentation_exc2( tstate, PY_MONITORING_EVENT_C_RAISE, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = _Py_call_instrumentation_2args( tstate, PY_MONITORING_EVENT_C_RETURN, frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (super == NULL) { @@ -10070,9 +10608,13 @@ } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr_o = PyObject_GetAttr(super, name); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attr_o == NULL) { JUMP_TO_LABEL(error); } @@ -10127,20 +10669,30 @@ STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = self_st; self_st = PyStackRef_NULL; stack_pointer[-1] = self_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-2] = class_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = global_super_st; global_super_st = PyStackRef_NULL; stack_pointer[-3] = global_super_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (attr == NULL) { @@ -10201,9 +10753,10 @@ { int *method_found_ptr = &method_found; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); attr_o = _PySuper_Lookup(cls, self, name, Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? method_found_ptr : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } if (attr_o == NULL) { JUMP_TO_LABEL(error); @@ -10214,23 +10767,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); self_or_null = PyStackRef_NULL; stack_pointer += 1; } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = global_super_st; global_super_st = self_or_null; stack_pointer[-2] = global_super_st; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = class_st; class_st = PyStackRef_NULL; stack_pointer[-1] = class_st; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); attr = PyStackRef_FromPyObjectSteal(attr_o); @@ -10258,8 +10816,9 @@ _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10280,9 +10839,10 @@ codeobj_st = stack_pointer[-1]; PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (func_obj == NULL) { JUMP_TO_LABEL(error); } @@ -10295,9 +10855,10 @@ { value = co; stack_pointer[-1] = func; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -10319,12 +10880,13 @@ PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); assert(PyDict_CheckExact(dict)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2( (PyDictObject *)dict, PyStackRef_AsPyObjectSteal(key), PyStackRef_AsPyObjectSteal(value) ); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(pop_2_error); } @@ -10356,11 +10918,12 @@ subject = stack_pointer[-3]; assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *attrs_o = _PyEval_MatchClass(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(type), oparg, PyStackRef_AsPyObjectBorrow(names)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (attrs_o) { assert(PyTuple_CheckExact(attrs_o)); attrs = PyStackRef_FromPyObjectSteal(attrs_o); @@ -10381,9 +10944,10 @@ stack_pointer[-3] = attrs; stack_pointer[-2] = s; stack_pointer[-1] = tp; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -10391,8 +10955,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -10400,8 +10965,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -10420,9 +10986,10 @@ keys = stack_pointer[-1]; subject = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (values_or_none_o == NULL) { JUMP_TO_LABEL(error); } @@ -10505,10 +11072,11 @@ exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XSETREF(exc_info->exc_value, PyStackRef_IsNone(exc_value) ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); @@ -10530,8 +11098,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10580,11 +11149,12 @@ else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_TRUE @@ -10623,11 +11193,12 @@ else { b = PyStackRef_False; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = value; value = b; stack_pointer[-1] = value; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } // _POP_JUMP_IF_FALSE @@ -10678,8 +11249,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -10746,12 +11318,17 @@ stack_pointer += -oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } JUMP_TO_LABEL(error); @@ -10780,8 +11357,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetRaisedException(tstate, exc); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); monitor_reraise(tstate, frame, this_instr); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } @@ -10815,9 +11399,10 @@ if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_CODEUNIT *bytecode = _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (bytecode == NULL) { JUMP_TO_LABEL(error); } @@ -10842,8 +11427,9 @@ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -10857,15 +11443,17 @@ uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Resume(this_instr, tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _CHECK_PERIODIC_IF_NOT_YIELD_FROM { if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err != 0) { JUMP_TO_LABEL(error); } @@ -10995,13 +11583,15 @@ assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (gen == NULL) { JUMP_TO_LABEL(error); } assert(STACK_LEVEL() <= 2); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyInterpreterFrame *gen_frame = &gen->gi_iframe; frame->instr_ptr++; _PyFrame_Copy(frame, gen_frame); @@ -11010,10 +11600,12 @@ gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_UpdateLastProfiledFrame(tstate, frame, prev); _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -11046,12 +11638,15 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(STACK_LEVEL() == 0); + DTRACE_FUNCTION_RETURN(); _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *dying = frame; frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); @@ -11086,8 +11681,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); @@ -11099,7 +11695,6 @@ v = stack_pointer[-1]; null_or_index = stack_pointer[-2]; PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!IS_PEP523_HOOKED(tstate) && (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && @@ -11118,10 +11713,11 @@ gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); } - if (!PyStackRef_IsNull(null_or_index)) { + if (!PyStackRef_IsNull(null_or_index) && PyStackRef_IsNone(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, receiver, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { JUMP_TO_LABEL(error); @@ -11133,51 +11729,93 @@ retval = item; } else { - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PySendResultPair res = _PyIter_Send(receiver_o, v_o); + _PyFrame_StackPointerInvalidate(frame); + if (res.kind == PYGEN_ERROR) { + JUMP_TO_LABEL(error); } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PyStackRef_CLOSE(v); + _PyFrame_StackPointerInvalidate(frame); + retval = PyStackRef_FromPyObjectSteal(res.object); + if (res.kind == PYGEN_RETURN) { + JUMPBY(oparg); } - retval = PyStackRef_FromPyObjectSteal(retval_o); + stack_pointer += 1; } - stack_pointer[-2] = null_or_index; - stack_pointer[-1] = retval; + } + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = retval; + DISPATCH(); + } + + TARGET(SEND_ASYNC_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_ASYNC_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_ASYNC_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_in; + _PyStackRef v; + _PyStackRef asend; + _PyStackRef null_out; + _PyStackRef retval; + /* Skip 1 cache entry */ + // _GUARD_3OS_ASYNC_GEN_ASEND + { + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (!PyAsyncGenASend_CheckExact(iter_o)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_ASYNC_GEN + { + v = stack_pointer[-1]; + null_in = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyAsyncGenASend_CheckExact(iter_o)); + PyObject *val = PyStackRef_AsPyObjectBorrow(v); + PyObject *retval_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + PySendResult what = _PyAsyncGenASend_Send(iter_o, val, &retval_o); + _PyFrame_StackPointerInvalidate(frame); + if (what == PYGEN_ERROR) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); + asend = iter; + null_out = null_in; + retval = PyStackRef_FromPyObjectSteal(retval_o); + if (what == PYGEN_RETURN) { + JUMPBY(oparg); + } } + stack_pointer[-2] = asend; + stack_pointer[-1] = null_out; + stack_pointer[0] = retval; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); } @@ -11238,17 +11876,88 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); assert(temp->previous == frame || temp->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; - LOAD_SP(); + stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(0); + #ifdef Py_DEBUG + assert(frame->previous->stackpointer_valid == 1); + #endif + DTRACE_FUNCTION_ENTRY(); LLTRACE_RESUME_FRAME(); } DISPATCH(); } + TARGET(SEND_VIRTUAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_VIRTUAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_VIRTUAL); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef val; + _PyStackRef nos; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef none; + _PyStackRef next; + /* Skip 1 cache entry */ + // _GUARD_TOS_IS_NONE + { + val = stack_pointer[-1]; + if (!PyStackRef_IsNone(val)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + if (PyStackRef_IsNull(nos)) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_VIRTUAL + { + none = val; + null_or_index = nos; + iter = stack_pointer[-3]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + Py_ssize_t index = PyStackRef_UntagInt(null_or_index); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + _PyObjectIndexPair next_index = Py_TYPE(iter_o)->_tp_iteritem(iter_o, index); + _PyFrame_StackPointerInvalidate(frame); + PyObject *next_o = next_index.object; + index = next_index.index; + if (next_o == NULL) { + if (index < 0) { + JUMP_TO_LABEL(error); + } + next = none; + JUMPBY(oparg); + DISPATCH(); + } + next = PyStackRef_FromPyObjectSteal(next_o); + null_or_index = PyStackRef_TagInt(index); + } + stack_pointer[-2] = null_or_index; + stack_pointer[-1] = next; + DISPATCH(); + } + TARGET(SETUP_ANNOTATIONS) { #if _Py_TAIL_CALL_INTERP int opcode = SETUP_ANNOTATIONS; @@ -11259,38 +11968,46 @@ INSTRUCTION_STATS(SETUP_ANNOTATIONS); if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } int err; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject* ann_dict = _PyMapping_GetOptionalItem2(LOCALS(), &_Py_ID(__annotations__), &err); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (ann_dict == NULL) { JUMP_TO_LABEL(error); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), ann_dict); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11308,9 +12025,10 @@ v = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(pop_1_error); } @@ -11364,9 +12082,10 @@ iterable = stack_pointer[-1]; set = stack_pointer[-2 - (oparg-1)]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -11378,8 +12097,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11407,8 +12127,9 @@ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_ATTR); @@ -11421,17 +12142,24 @@ v = stack_pointer[-2]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), name, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = owner; owner = PyStackRef_NULL; stack_pointer[-1] = owner; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-2] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -11517,8 +12245,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11526,8 +12255,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11579,8 +12309,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11588,8 +12319,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11668,8 +12400,9 @@ } } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyDict_NotifyEvent(PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); UNLOCK_OBJECT(dict); STAT_INC(STORE_ATTR, hit); @@ -11678,8 +12411,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP { @@ -11687,8 +12421,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11705,8 +12440,9 @@ v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); DISPATCH(); @@ -11735,8 +12471,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -11759,8 +12496,9 @@ value2 = PyStackRef_DUP(GETLOCAL(oparg2)); stack_pointer[-1] = value2; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -11783,15 +12521,17 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); tmp = GETLOCAL(oparg2); GETLOCAL(oparg2) = value2; stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH(); } @@ -11807,13 +12547,15 @@ v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -11835,31 +12577,36 @@ int err; if (ns == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_Format(tstate, PyExc_SystemError, "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } else { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { JUMP_TO_LABEL(error); } @@ -11891,9 +12638,11 @@ container = stack_pointer[-3]; v = stack_pointer[-4]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); + PyStackRef_AsPyObjectSteal(stop), + Py_None); + _PyFrame_StackPointerInvalidate(frame); int err; if (slice == NULL) { err = 1; @@ -11902,21 +12651,29 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += 2; } _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = container; container = PyStackRef_NULL; stack_pointer[-3] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-4] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -4; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -11950,8 +12707,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); @@ -11962,20 +12720,30 @@ { v = stack_pointer[-3]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyStackRef tmp = sub; sub = PyStackRef_NULL; stack_pointer[-1] = sub; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = container; container = PyStackRef_NULL; stack_pointer[-2] = container; PyStackRef_CLOSE(tmp); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); tmp = v; v = PyStackRef_NULL; stack_pointer[-3] = v; PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); if (err) { @@ -12026,16 +12794,18 @@ assert(Py_TYPE(dict)->tp_as_mapping->mp_ass_subscript == _PyDict_StoreSubscript); STAT_INC(STORE_SUBSCR, hit); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = _PyDict_SetItem_Take2((PyDictObject *)dict, PyStackRef_AsPyObjectSteal(sub), PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err) { stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } st = dict_st; @@ -12046,8 +12816,9 @@ stack_pointer += -3; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12130,8 +12901,9 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } // _POP_TOP_INT { @@ -12145,8 +12917,9 @@ stack_pointer += -2; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12193,8 +12966,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); @@ -12205,13 +12979,15 @@ // _TO_BOOL { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } @@ -12262,8 +13038,9 @@ value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12376,8 +13153,9 @@ value = v; stack_pointer[-1] = res; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12481,30 +13259,34 @@ _PyJitTracerState *tracer = _tstate->jit_tracer_state; assert(tracer != NULL); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing ? _DEOPT : 0); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (full) { LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } DISPATCH(); } for (int i = 0; i < tracer->prev_state.recorded_count; i++) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_CLEAR(tracer->prev_state.recorded_values[i]); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.recorded_count = 0; tracer->prev_state.instr = next_instr; PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); if (tracer->prev_state.instr_code != (PyCodeObject *)prev_code) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_SETREF(tracer->prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } tracer->prev_state.instr_frame = frame; tracer->prev_state.instr_oparg = oparg; @@ -12543,8 +13325,9 @@ { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12555,9 +13338,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12577,8 +13361,9 @@ { value = stack_pointer[-1]; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12589,9 +13374,10 @@ { value = v; stack_pointer[-1] = res; - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12630,9 +13416,13 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } @@ -12663,8 +13453,9 @@ if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); @@ -12680,9 +13471,13 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res == 0) { JUMP_TO_LABEL(error); } @@ -12745,8 +13540,9 @@ stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12795,8 +13591,9 @@ stack_pointer += -1 + oparg; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12846,8 +13643,9 @@ stack_pointer += 1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } DISPATCH(); } @@ -12885,13 +13683,15 @@ PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; int has_self = !PyStackRef_IsNull(exit_self); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (res_o == NULL) { JUMP_TO_LABEL(error); } @@ -12910,6 +13710,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); + opcode = YIELD_VALUE; _PyStackRef value; _PyStackRef retval; // _MAKE_HEAP_SAFE @@ -12929,24 +13730,28 @@ stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + DTRACE_FUNCTION_RETURN(); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; + _PyThreadState_UpdateLastProfiledFrame(tstate, gen_frame, gen_frame->previous); frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; ((_PyThreadStateImpl *)tstate)->generator_return_kind = GENERATOR_YIELD; FT_ATOMIC_STORE_INT8_RELEASE(gen->gi_frame_state, FRAME_SUSPENDED + oparg); assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + Py_ssize_t i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + assert(i >= 0 && i <= INT_MAX); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), (int)i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = temp; LLTRACE_RESUME_FRAME(); @@ -12999,32 +13804,42 @@ JUMP_TO_LABEL(error); LABEL(error) { + _PyFrame_StackAssertInvalid(frame); #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyErr_SetString(tstate, PyExc_SystemError, "error return without exception set"); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } #else assert(_PyErr_Occurred(tstate)); #endif _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); STOP_TRACING(); stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); if (!_PyFrame_IsIncomplete(frame)) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyFrameObject *f = _PyFrame_GetFrameObject(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (f != NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); PyTraceBack_Here(f); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); } } - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); _PyEval_MonitorRaise(tstate, frame, next_instr-1); + _PyFrame_StackPointerInvalidate(frame); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); JUMP_TO_LABEL(exception_unwind); } @@ -13069,6 +13884,7 @@ JUMP_TO_LABEL(error); } #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif @@ -13076,6 +13892,13 @@ JUMP_TO_LABEL(error); } LABEL(exit_unwind) + { + assert(_PyErr_Occurred(tstate)); + DTRACE_FUNCTION_RETURN(); + JUMP_TO_LABEL(exit_unwind_notrace); + } + + LABEL(exit_unwind_notrace) { assert(_PyErr_Occurred(tstate)); _Py_LeaveRecursiveCallPy(tstate); @@ -13101,15 +13924,20 @@ JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } LABEL(start_frame) { + #ifdef Py_DEBUG + assert(frame->stackpointer_valid == 1); + #endif int too_deep = _Py_EnterRecursivePy(tstate); if (too_deep) { - JUMP_TO_LABEL(exit_unwind); + JUMP_TO_LABEL(exit_unwind_notrace); } + DTRACE_FUNCTION_ENTRY(); next_instr = frame->instr_ptr; #ifdef Py_DEBUG int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS()); @@ -13120,6 +13948,7 @@ JUMP_TO_LABEL(error); assert(!_PyErr_Occurred(tstate)); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); #if _Py_TAIL_CALL_INTERP int opcode; #endif @@ -13135,12 +13964,14 @@ JUMP_TO_LABEL(error); assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, _EXIT_TRACE); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); int err = stop_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); + _PyFrame_StackPointerInvalidate(frame); if (err < 0) { JUMP_TO_LABEL(error); } diff --git a/Python/hamt.c b/Python/hamt.c index e4719e71a5259a5..95998ae5062ac7e 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -702,6 +702,7 @@ hamt_node_bitmap_assoc(PyHamtNode_Bitmap *self, PyHamtNode_Bitmap *ret = hamt_node_bitmap_clone(self); if (ret == NULL) { + Py_DECREF(sub_node); return NULL; } Py_SETREF(ret->b_array[val_idx], (PyObject*)sub_node); @@ -994,6 +995,7 @@ hamt_node_bitmap_without(PyHamtNode_Bitmap *self, PyHamtNode_Bitmap *clone = hamt_node_bitmap_clone(self); if (clone == NULL) { + Py_DECREF(sub_node); return W_ERROR; } diff --git a/Python/import.c b/Python/import.c index 7aa96196ec1e105..6da6faf5f28cc3b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -81,8 +81,6 @@ static struct _inittab *inittab_copy = NULL; #define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index #define EXTENSIONS _PyRuntime.imports.extensions -#define PKGCONTEXT (_PyRuntime.imports.pkgcontext) - /*******************************/ /* interpreter import state */ @@ -94,6 +92,8 @@ static struct _inittab *inittab_copy = NULL; (interp)->imports.modules_by_index #define LAZY_MODULES(interp) \ (interp)->imports.lazy_modules +#define LAZY_PENDING_SUBMODULES(interp) \ + (interp)->imports.lazy_pending_submodules #define IMPORTLIB(interp) \ (interp)->imports.importlib #define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \ @@ -271,8 +271,11 @@ import_get_module(PyThreadState *tstate, PyObject *name) PyObject * _PyImport_InitLazyModules(PyInterpreterState *interp) { - assert(LAZY_MODULES(interp) == NULL); - LAZY_MODULES(interp) = PyDict_New(); + assert(LAZY_MODULES(interp) == NULL && + LAZY_PENDING_SUBMODULES(interp) == NULL); + + LAZY_PENDING_SUBMODULES(interp) = PyDict_New(); + LAZY_MODULES(interp) = PySet_New(0); return LAZY_MODULES(interp); } @@ -280,6 +283,7 @@ void _PyImport_ClearLazyModules(PyInterpreterState *interp) { Py_CLEAR(LAZY_MODULES(interp)); + Py_CLEAR(LAZY_PENDING_SUBMODULES(interp)); } static int @@ -877,7 +881,6 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) */ static _Py_thread_local const char *pkgcontext = NULL; -# undef PKGCONTEXT # define PKGCONTEXT pkgcontext const char * @@ -2059,7 +2062,7 @@ import_run_modexport(PyThreadState *tstate, PyModExportFunction ex0, /* This is like import_run_extension, but avoids interpreter switching * and code for for single-phase modules. */ - PyModuleDef_Slot *slots = ex0(); + PySlot *slots = ex0(); if (!slots) { if (!PyErr_Occurred()) { PyErr_Format( @@ -2162,6 +2165,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, struct _Py_ext_module_loader_result res; int rc = _PyImport_RunModInitFunc(p0, info, &res); + bool main_error = false; if (rc < 0) { /* We discard res.def. */ assert(res.module == NULL); @@ -2186,7 +2190,8 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, // obmalloc, so we create a copy here. filename = _PyUnicode_Copy(info->filename); if (filename == NULL) { - return NULL; + main_error = true; + goto main_finally; } } else { @@ -2232,6 +2237,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, main_tstate, info->path, info->name, def, &singlephase); if (cached == NULL) { assert(PyErr_Occurred()); + main_error = true; goto main_finally; } } @@ -2247,7 +2253,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, // gh-144601: The exception object can't be transferred across // interpreters. Instead, we print out an unraisable exception, and // then raise a different exception for the calling interpreter. - if (rc < 0) { + if (rc < 0 || main_error) { assert(PyErr_Occurred()); PyErr_FormatUnraisable("Exception while importing from subinterpreter"); } @@ -2256,7 +2262,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, /* Any module we got from the init function will have to be * reloaded in the subinterpreter. */ mod = NULL; - if (rc < 0) { + if (rc < 0 || main_error) { PyErr_SetString(PyExc_ImportError, "failed to import from subinterpreter due to exception"); goto error; @@ -2271,6 +2277,9 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, if (rc < 0) { goto error; } + if (main_error) { + goto error; + } if (res.kind == _Py_ext_module_kind_MULTIPHASE) { assert_multiphase_def(def); @@ -3162,7 +3171,7 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) if (nameobj == NULL || nameobj == Py_None) { return FROZEN_BAD_NAME; } - const char *name = PyUnicode_AsUTF8(nameobj); + const char *name = _PyUnicode_AsUTF8NoNUL(nameobj); if (name == NULL) { // Note that this function previously used // _PyUnicode_EqualToASCIIString(). We clear the error here @@ -3928,7 +3937,6 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) return NULL; } else if (PySet_Add(importing, lazy_import) < 0) { - _PyImport_ReleaseLock(interp); goto error; } @@ -4049,7 +4057,7 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) // Create a cause exception showing where the lazy import was declared. PyObject *msg = PyUnicode_FromFormat( - "deferred import of '%U' raised an exception during resolution", + "lazy import of '%U' raised an exception during resolution", import_name ); Py_DECREF(import_name); // Done with import_name. @@ -4326,20 +4334,10 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, return final_mod; } -static PyObject * -get_mod_dict(PyObject *module) -{ - if (PyModule_Check(module)) { - return Py_NewRef(_PyModule_GetDict(module)); - } - - return PyObject_GetAttr(module, &_Py_ID(__dict__)); -} - // ensure we have the set for the parent module name in sys.lazy_modules. // Returns a new reference. static PyObject * -ensure_lazy_submodules(PyDictObject *lazy_modules, PyObject *parent) +ensure_lazy_pending_submodules(PyDictObject *lazy_modules, PyObject *parent) { PyObject *lazy_submodules; Py_BEGIN_CRITICAL_SECTION(lazy_modules); @@ -4358,27 +4356,28 @@ ensure_lazy_submodules(PyDictObject *lazy_modules, PyObject *parent) return lazy_submodules; } +// Records all parent-child relationships in lazy_pending_submodules +// for a lazily imported module name. When a parent module's attribute +// is accessed, _Py_module_getattro_impl will check lazy_pending_submodules +// and trigger the import. static int -register_lazy_on_parent(PyThreadState *tstate, PyObject *name, - PyObject *builtins) +register_lazy_on_parent(PyThreadState *tstate, PyObject *name) { int ret = -1; PyObject *parent = NULL; PyObject *child = NULL; - PyObject *parent_module = NULL; - PyObject *parent_dict = NULL; PyInterpreterState *interp = tstate->interp; - PyObject *lazy_modules = LAZY_MODULES(interp); - assert(lazy_modules != NULL); + PyObject *lazy_pending_submodules = LAZY_PENDING_SUBMODULES(interp); + assert(lazy_pending_submodules != NULL); Py_INCREF(name); while (true) { Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0, PyUnicode_GET_LENGTH(name), -1); if (dot < 0) { - PyObject *lazy_submodules = ensure_lazy_submodules( - (PyDictObject *)lazy_modules, name); + PyObject *lazy_submodules = ensure_lazy_pending_submodules( + (PyDictObject *)lazy_pending_submodules, name); if (lazy_submodules == NULL) { goto done; } @@ -4387,9 +4386,6 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, goto done; } parent = PyUnicode_Substring(name, 0, dot); - // If `parent` is NULL then this has hit the end of the import, no - // more "parent.child" in the import name. The entire import will be - // resolved lazily. if (parent == NULL) { goto done; } @@ -4399,9 +4395,8 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, goto done; } - // Record the child as being lazily imported from the parent. - PyObject *lazy_submodules = ensure_lazy_submodules( - (PyDictObject *)lazy_modules, parent); + PyObject *lazy_submodules = ensure_lazy_pending_submodules( + (PyDictObject *)lazy_pending_submodules, parent); if (lazy_submodules == NULL) { goto done; } @@ -4412,44 +4407,11 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, } Py_DECREF(lazy_submodules); - // Add the lazy import for the child to the parent. - Py_XSETREF(parent_module, PyImport_GetModule(parent)); - if (parent_module != NULL) { - Py_XSETREF(parent_dict, get_mod_dict(parent_module)); - if (parent_dict == NULL) { - goto done; - } - if (PyDict_CheckExact(parent_dict)) { - int contains = PyDict_Contains(parent_dict, child); - if (contains < 0) { - goto done; - } - if (!contains) { - PyObject *lazy_module_attr = _PyLazyImport_New( - tstate->current_frame, builtins, parent, child - ); - if (lazy_module_attr == NULL) { - goto done; - } - if (PyDict_SetItem(parent_dict, child, - lazy_module_attr) < 0) { - Py_DECREF(lazy_module_attr); - goto done; - } - Py_DECREF(lazy_module_attr); - } - } - ret = 0; - goto done; - } - Py_SETREF(name, parent); parent = NULL; } done: - Py_XDECREF(parent_dict); - Py_XDECREF(parent_module); Py_XDECREF(child); Py_XDECREF(parent); Py_XDECREF(name); @@ -4458,17 +4420,73 @@ register_lazy_on_parent(PyThreadState *tstate, PyObject *name, static int register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, - PyObject *from, PyObject *builtins) + PyObject *from) { PyObject *fromname = PyUnicode_FromFormat("%U.%U", abs_name, from); if (fromname == NULL) { return -1; } - int res = register_lazy_on_parent(tstate, fromname, builtins); + + // Add the module name to sys.lazy_modules set (PEP 810). + PyObject *lazy_modules = LAZY_MODULES(tstate->interp); + if (PySet_Add(lazy_modules, fromname) < 0) { + Py_DECREF(fromname); + return -1; + } + + int res = register_lazy_on_parent(tstate, fromname); Py_DECREF(fromname); return res; } +PyObject * +_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); + if (lazy_pending == NULL) { + return NULL; + } + + PyObject *pending_set; + int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); + if (rc <= 0) { + return NULL; + } + + int contains = PySet_Contains(pending_set, attr_name); + if (contains <= 0) { + Py_DECREF(pending_set); + return NULL; + } + + PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); + if (full_name == NULL) { + Py_DECREF(pending_set); + return NULL; + } + + PyObject *mod = PyImport_ImportModuleLevelObject( + full_name, NULL, NULL, NULL, 0); + if (mod == NULL) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return NULL; + } + Py_DECREF(mod); + + if (PySet_Discard(pending_set, attr_name) < 0) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return NULL; + } + Py_DECREF(pending_set); + + PyObject *submod = PyImport_GetModule(full_name); + Py_DECREF(full_name); + return submod; +} + PyObject * _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyObject *name, PyObject *builtins, @@ -4555,9 +4573,15 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, Py_DECREF(abs_name); return NULL; } + + // Add the module name to sys.lazy_modules set (PEP 810). + PyObject *lazy_modules = LAZY_MODULES(tstate->interp); + if (PySet_Add(lazy_modules, abs_name) < 0) { + goto error; + } + if (fromlist && PyUnicode_Check(fromlist)) { - if (register_from_lazy_on_parent(tstate, abs_name, fromlist, - builtins) < 0) { + if (register_from_lazy_on_parent(tstate, abs_name, fromlist) < 0) { goto error; } } @@ -4565,14 +4589,13 @@ _PyImport_LazyImportModuleLevelObject(PyThreadState *tstate, PyTuple_GET_SIZE(fromlist)) { for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(fromlist); i++) { if (register_from_lazy_on_parent(tstate, abs_name, - PyTuple_GET_ITEM(fromlist, i), - builtins) < 0) + PyTuple_GET_ITEM(fromlist, i)) < 0) { goto error; } } } - else if (register_lazy_on_parent(tstate, abs_name, builtins) < 0) { + else if (register_lazy_on_parent(tstate, abs_name) < 0) { goto error; } @@ -4791,6 +4814,7 @@ _PyImport_ClearCore(PyInterpreterState *interp) Py_CLEAR(IMPORTLIB(interp)); Py_CLEAR(IMPORT_FUNC(interp)); Py_CLEAR(LAZY_IMPORT_FUNC(interp)); + Py_CLEAR(interp->imports.lazy_pending_submodules); Py_CLEAR(interp->imports.lazy_modules); Py_CLEAR(interp->imports.lazy_importing_modules); Py_CLEAR(interp->imports.lazy_imports_filter); @@ -5003,18 +5027,18 @@ _imp_lock_held_impl(PyObject *module) } /*[clinic input] -@permit_long_docstring_body _imp.acquire_lock Acquires the interpreter's import lock for the current thread. -This lock should be used by import hooks to ensure thread-safety when importing -modules. On platforms without threads, this function does nothing. +This lock should be used by import hooks to ensure thread-safety when +importing modules. On platforms without threads, this function does +nothing. [clinic start generated code]*/ static PyObject * _imp_acquire_lock_impl(PyObject *module) -/*[clinic end generated code: output=1aff58cb0ee1b026 input=e1a4ef049d34e7dd]*/ +/*[clinic end generated code: output=1aff58cb0ee1b026 input=60e9c1b4ab471ead]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyImport_AcquireLock(interp); @@ -5172,6 +5196,7 @@ _imp_init_frozen_impl(PyObject *module, PyObject *name) } /*[clinic input] +@permit_long_summary _imp.find_frozen name: unicode @@ -5192,7 +5217,7 @@ The returned info (a 2-tuple): static PyObject * _imp_find_frozen_impl(PyObject *module, PyObject *name, int withdata) -/*[clinic end generated code: output=8c1c3c7f925397a5 input=22a8847c201542fd]*/ +/*[clinic end generated code: output=8c1c3c7f925397a5 input=30a7a50da49eca97]*/ { struct frozen_info info; frozen_status status = find_frozen(name, &info); @@ -5379,6 +5404,7 @@ _imp__override_frozen_modules_for_tests_impl(PyObject *module, int override) } /*[clinic input] +@permit_long_summary _imp._override_multi_interp_extensions_check override: int @@ -5392,7 +5418,7 @@ _imp._override_multi_interp_extensions_check static PyObject * _imp__override_multi_interp_extensions_check_impl(PyObject *module, int override) -/*[clinic end generated code: output=3ff043af52bbf280 input=e086a2ea181f92ae]*/ +/*[clinic end generated code: output=3ff043af52bbf280 input=24f23f8510a7f6e7]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); if (_Py_IsMainInterpreter(interp)) { @@ -5580,46 +5606,6 @@ _imp_source_hash_impl(PyObject *module, long key, Py_buffer *source) return PyBytes_FromStringAndSize(hash.data, sizeof(hash.data)); } -static int -publish_lazy_imports_on_module(PyThreadState *tstate, - PyObject *lazy_submodules, - PyObject *name, - PyObject *module_dict) -{ - PyObject *builtins = _PyEval_GetBuiltins(tstate); - PyObject *attr_name; - Py_ssize_t pos = 0; - Py_hash_t hash; - - // Enumerate the set of lazy submodules which have been imported from the - // parent module. - while (_PySet_NextEntryRef(lazy_submodules, &pos, &attr_name, &hash)) { - if (_PyDict_Contains_KnownHash(module_dict, attr_name, hash)) { - Py_DECREF(attr_name); - continue; - } - // Create a new lazy module attr for the subpackage which was - // previously lazily imported. - PyObject *lazy_module_attr = _PyLazyImport_New(tstate->current_frame, builtins, - name, attr_name); - if (lazy_module_attr == NULL) { - Py_DECREF(attr_name); - return -1; - } - - // Publish on the module that was just imported. - if (PyDict_SetItem(module_dict, attr_name, - lazy_module_attr) < 0) { - Py_DECREF(lazy_module_attr); - Py_DECREF(attr_name); - return -1; - } - Py_DECREF(lazy_module_attr); - Py_DECREF(attr_name); - } - return 0; -} - /*[clinic input] _imp._set_lazy_attributes modobj: object @@ -5633,43 +5619,11 @@ _imp__set_lazy_attributes_impl(PyObject *module, PyObject *modobj, PyObject *name) /*[clinic end generated code: output=3369bb3242b1f043 input=38ea6f30956dd7d6]*/ { - PyThreadState *tstate = _PyThreadState_GET(); - PyObject *module_dict = NULL; - PyObject *ret = NULL; - PyObject *lazy_modules = LAZY_MODULES(tstate->interp); - assert(lazy_modules != NULL); - - PyObject *lazy_submodules; - if (PyDict_GetItemRef(lazy_modules, name, &lazy_submodules) < 0) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (PySet_Discard(LAZY_MODULES(interp), name) < 0) { return NULL; } - else if (lazy_submodules == NULL) { - Py_RETURN_NONE; - } - - module_dict = get_mod_dict(modobj); - if (module_dict == NULL || !PyDict_CheckExact(module_dict)) { - Py_DECREF(lazy_submodules); - goto done; - } - - assert(PyAnySet_CheckExact(lazy_submodules)); - Py_BEGIN_CRITICAL_SECTION(lazy_submodules); - publish_lazy_imports_on_module(tstate, lazy_submodules, name, module_dict); - Py_END_CRITICAL_SECTION(); - Py_DECREF(lazy_submodules); - - // once a module is imported it is removed from sys.lazy_modules - if (PyDict_DelItem(lazy_modules, name) < 0) { - goto error; - } - -done: - ret = Py_NewRef(Py_None); - -error: - Py_XDECREF(module_dict); - return ret; + Py_RETURN_NONE; } PyDoc_STRVAR(doc_imp, diff --git a/Python/initconfig.c b/Python/initconfig.c index 8dc9602ff13df79..f653c00d1f0bed8 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -250,224 +250,340 @@ static void initconfig_free_config(const PyConfig *config); /* --- Command line options --------------------------------------- */ -/* Short usage message (with %s for argv0) */ +/* + * Help text markup (matching Lib/_colorize.py Argparse theme). + * + * Color spans, #X{...} where "}" resets to default color: + * #b{...} label bold yellow + * #B{...} summary label yellow + * #E{...} env var (primary) bold cyan + * #e{...} env var reference cyan + * #h{...} heading bold blue + * #L{...} long option bold cyan + * #s{...} short option bold green + * #S{...} summary short opt green + * + * Runtime substitutions (no "{" follows): + * #P program name (bold magenta) + * #D path separator (DELIM) + * #H PYTHONHOMEHELP default search path + * + * fprint_help() walks the string, expanding color codes only when colorize=1 + * and substituting runtime values regardless. + */ + +#if defined(MS_WINDOWS) +# define PYTHONHOMEHELP "<prefix>\\python{major}{minor}" +#else +# define PYTHONHOMEHELP "<prefix>/lib/pythonX.X" +#endif + +/* Determine if we can emit ANSI color codes on the given stream. + * Logic mirrors Lib/_colorize.py:can_colorize(). */ +static int +_Py_can_colorize(FILE *f) +{ + const char *env; + + env = Py_GETENV("PYTHON_COLORS"); + if (env) { + if (strcmp(env, "0") == 0) { + return 0; + } + if (strcmp(env, "1") == 0) { + return 1; + } + } + if (getenv("NO_COLOR")) { + return 0; + } + if (getenv("FORCE_COLOR")) { + return 1; + } + env = getenv("TERM"); + if (env && strcmp(env, "dumb") == 0) { + return 0; + } +#if defined(MS_WINDOWS) && defined(HAVE_WINDOWS_CONSOLE_IO) + { + DWORD mode = 0; + DWORD nStdHandle = (f == stderr) ? STD_ERROR_HANDLE + : STD_OUTPUT_HANDLE; + HANDLE handle = GetStdHandle(nStdHandle); + if (!GetConsoleMode(handle, &mode) + || !(mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING)) + { + return 0; + } + } +#endif + return isatty(fileno(f)); +} + +/* Walk help text, expanding markup: + * #X{...} color span (only emitted when colorize=1; '}' resets). + * #X runtime substitution (program name, DELIM, PYTHONHOMEHELP). + * See the markup table above the macro/comment block. */ +static void +fprint_help(FILE *f, const char *text, int colorize, const wchar_t *program) +{ + for (const char *p = text; *p; ) { + if (*p == '#' && p[1]) { + char code = p[1]; + if (p[2] == '{') { + /* Color span open */ + const char *seq = NULL; + switch (code) { + case 'h': seq = "\x1b[1;34m"; break; // heading + case 'E': seq = "\x1b[1;36m"; break; // env var primary + case 'e': seq = "\x1b[36m"; break; // env var reference + case 'L': seq = "\x1b[1;36m"; break; // long option + case 'b': seq = "\x1b[1;33m"; break; // label + case 'B': seq = "\x1b[33m"; break; // summary label + case 's': seq = "\x1b[1;32m"; break; // short option + case 'S': seq = "\x1b[32m"; break; // summary short option + } + if (colorize && seq) fputs(seq, f); + p += 3; // skip "#X{" + continue; + } + /* Runtime substitution */ + switch (code) { + case 'P': // program name with bold magenta + if (colorize) fputs("\x1b[1;35m", f); + if (program) fprintf(f, "%ls", program); + if (colorize) fputs("\x1b[0m", f); + break; + case 'D': + fputc((char)DELIM, f); + break; + case 'H': + fputs(PYTHONHOMEHELP, f); + break; + default: // unknown: emit literally + fputc('#', f); + fputc(code, f); + break; + } + p += 2; // skip "#X" + continue; + } + if (*p == '}') { + if (colorize) fputs("\x1b[0m", f); + p++; + continue; + } + fputc(*p++, f); + } +} + +/* Short usage message */ static const char usage_line[] = -"usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; +"#h{usage:} #P [#S{option}] #S{...} " +"[#S{-c} #B{cmd} | #S{-m} #B{mod} | #S{file} | #S{-}] " +"[#S{arg}] #S{...}\n" +; /* Long help message */ /* Lines sorted by option name; keep in sync with usage_envvars* below */ -static const char usage_help[] = "\ -Options (and corresponding environment variables):\n\ --b : issue warnings about converting bytes/bytearray to str and comparing\n\ - bytes/bytearray with str or bytes with int. (-bb: issue errors)\n\ - deprecated since 3.15 and will become no-op in 3.17.\n\ --B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ --c cmd : program passed in as string (terminates option list)\n\ --d : turn on parser debugging output (for experts only, only works on\n\ - debug builds); also PYTHONDEBUG=x\n\ --E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ --h : print this help message and exit (also -? or --help)\n\ --i : inspect interactively after running script; forces a prompt even\n\ - if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ --I : isolate Python from the user's environment (implies -E, -P and -s)\n\ --m mod : run library module as a script (terminates option list)\n\ --O : remove assert and __debug__-dependent statements; add .opt-1 before\n\ - .pyc extension; also PYTHONOPTIMIZE=x\n\ --OO : do -O changes and also discard docstrings; add .opt-2 before\n\ - .pyc extension\n\ --P : don't prepend a potentially unsafe path to sys.path; also\n\ - PYTHONSAFEPATH\n\ --q : don't print version and copyright messages on interactive startup\n\ --s : don't add user site directory to sys.path; also PYTHONNOUSERSITE=x\n\ --S : don't imply 'import site' on initialization\n\ --u : force the stdout and stderr streams to be unbuffered;\n\ - this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ --v : verbose (trace import statements); also PYTHONVERBOSE=x\n\ - can be supplied multiple times to increase verbosity\n\ --V : print the Python version number and exit (also --version)\n\ - when given twice, print more information about the build\n\ --W arg : warning control; arg is action:message:category:module:lineno\n\ - also PYTHONWARNINGS=arg\n\ --x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ --X opt : set implementation-specific option\n\ ---check-hash-based-pycs always|default|never:\n\ - control how Python invalidates hash-based .pyc files\n\ ---help-env: print help about Python environment variables and exit\n\ ---help-xoptions: print help about implementation-specific -X options and exit\n\ ---help-all: print complete help information and exit\n\ -\n\ -Arguments:\n\ -file : program read from script file\n\ -- : program read from stdin (default; interactive mode if a tty)\n\ -arg ...: arguments passed to program in sys.argv[1:]\n\ -"; - -static const char usage_xoptions[] = "\ -The following implementation-specific options are available:\n\ --X context_aware_warnings=[0|1]: if true (1) then the warnings module will\n\ - use a context variables; if false (0) then the warnings module will\n\ - use module globals, which is not concurrent-safe; set to true for\n\ - free-threaded builds and false otherwise; also\n\ - PYTHON_CONTEXT_AWARE_WARNINGS\n\ --X cpu_count=N: override the return value of os.cpu_count();\n\ - -X cpu_count=default cancels overriding; also PYTHON_CPU_COUNT\n\ --X dev : enable Python Development Mode; also PYTHONDEVMODE\n\ --X disable-remote-debug: disable remote debugging; also PYTHON_DISABLE_REMOTE_DEBUG\n\ --X faulthandler: dump the Python traceback on fatal errors;\n\ - also PYTHONFAULTHANDLER\n\ --X frozen_modules=[on|off]: whether to use frozen modules; the default is \"on\"\n\ - for installed Python and \"off\" for a local build;\n\ - also PYTHON_FROZEN_MODULES\n\ -" +static const char usage_help[] = +"#h{Options (and corresponding environment variables):}\n" +"#s{-b} : issue warnings about converting bytes/bytearray to str and comparing\n" +" bytes/bytearray with str or bytes with int. (#S{-bb}: issue errors)\n" +" deprecated since 3.15 and will become no-op in 3.17.\n" +"#s{-B} : don't write .pyc files on import; also #e{PYTHONDONTWRITEBYTECODE}#B{=x}\n" +"#s{-c} #b{cmd} : program passed in as string (terminates option list)\n" +"#s{-d} : turn on parser debugging output (for experts only, only works on\n" +" debug builds); also #e{PYTHONDEBUG}#B{=x}\n" +"#s{-E} : ignore #e{PYTHON*} environment variables (such as #e{PYTHONPATH})\n" +"#s{-h} : print this help message and exit (also #S{-?} or #e{--help})\n" +"#s{-i} : inspect interactively after running script; forces a prompt even\n" +" if stdin does not appear to be a terminal; also #e{PYTHONINSPECT}#B{=x}\n" +"#s{-I} : isolate Python from the user's environment (implies #S{-E}, #S{-P} and #S{-s})\n" +"#s{-m} #b{mod} : run library module as a script (terminates option list)\n" +"#s{-O} : remove assert and __debug__-dependent statements; add .opt-1 before\n" +" .pyc extension; also #e{PYTHONOPTIMIZE}#B{=x}\n" +"#s{-OO} : do #S{-O} changes and also discard docstrings; add .opt-2 before\n" +" .pyc extension\n" +"#s{-P} : don't prepend a potentially unsafe path to sys.path; also\n" +" #e{PYTHONSAFEPATH}\n" +"#s{-q} : don't print version and copyright messages on interactive startup\n" +"#s{-s} : don't add user site directory to sys.path; also #e{PYTHONNOUSERSITE}#B{=x}\n" +"#s{-S} : don't imply 'import site' on initialization\n" +"#s{-u} : force the stdout and stderr streams to be unbuffered;\n" +" this option has no effect on stdin; also #e{PYTHONUNBUFFERED}#B{=x}\n" +"#s{-v} : verbose (trace import statements); also #e{PYTHONVERBOSE}#B{=x}\n" +" can be supplied multiple times to increase verbosity\n" +"#s{-V} : print the Python version number and exit (also #e{--version})\n" +" when given twice, print more information about the build\n" +"#s{-W} #b{arg} : warning control; #B{arg} is action:message:category:module:lineno\n" +" also #e{PYTHONWARNINGS}#B{=arg}\n" +"#s{-x} : skip first line of source, allowing use of non-Unix forms of #!cmd\n" +"#s{-X} #b{opt} : set implementation-specific option\n" +"#L{--check-hash-based-pycs} #b{always|default|never}:\n" +" control how Python invalidates hash-based .pyc files\n" +"#L{--help-env}: print help about Python environment variables and exit\n" +"#L{--help-xoptions}: print help about implementation-specific #S{-X} options and exit\n" +"#L{--help-all}: print complete help information and exit\n" +"\n" +"#h{Arguments:}\n" +"#s{file} : program read from script file\n" +"#s{-} : program read from stdin (default; interactive mode if a tty)\n" +"#s{arg} #b{...}: arguments passed to program in sys.argv[1:]\n" +; + +static const char usage_xoptions[] = +"#h{The following implementation-specific options are available:}\n" +"#s{-X} #L{context_aware_warnings}#b{=[0|1]}: if true (#B{1}) then the warnings module will\n" +" use a context variables; if false (#B{0}) then the warnings module will\n" +" use module globals, which is not concurrent-safe; set to true for\n" +" free-threaded builds and false otherwise; also\n" +" #e{PYTHON_CONTEXT_AWARE_WARNINGS}\n" +"#s{-X} #L{cpu_count}#b{=N}: override the return value of os.cpu_count();\n" +" #S{-X} #e{cpu_count}#B{=default} cancels overriding; also #e{PYTHON_CPU_COUNT}\n" +"#s{-X} #L{dev} : enable Python Development Mode; also #e{PYTHONDEVMODE}\n" +"#s{-X} #L{disable-remote-debug}: disable remote debugging; also #e{PYTHON_DISABLE_REMOTE_DEBUG}\n" +"#s{-X} #L{faulthandler}: dump the Python traceback on fatal errors;\n" +" also #e{PYTHONFAULTHANDLER}\n" +"#s{-X} #L{frozen_modules}#b{=[on|off]}: whether to use frozen modules; the default is \"#B{on}\"\n" +" for installed Python and \"#B{off}\" for a local build;\n" +" also #e{PYTHON_FROZEN_MODULES}\n" #ifdef Py_GIL_DISABLED -"-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" +"#s{-X} #L{gil}#b{=[0|1]}: enable (#B{1}) or disable (#B{0}) the GIL; also #e{PYTHON_GIL}\n" #endif -"\ --X importtime[=2]: show how long each import takes; use -X importtime=2 to\n\ - log imports of already-loaded modules; also PYTHONPROFILEIMPORTTIME\n\ --X int_max_str_digits=N: limit the size of int<->str conversions;\n\ - 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ --X lazy_imports=[all|none|normal]: control global lazy imports;\n\ - default is normal; also PYTHON_LAZY_IMPORTS\n\ --X no_debug_ranges: don't include extra location information in code objects;\n\ - also PYTHONNODEBUGRANGES\n\ --X pathconfig_warnings=[0|1]: if true (1) then path configuration is allowed\n\ - to log warnings into stderr; if false (0) suppress these warnings;\n\ - set to true by default; also PYTHON_PATHCONFIG_WARNINGS\n\ --X perf: support the Linux \"perf\" profiler; also PYTHONPERFSUPPORT=1\n\ --X perf_jit: support the Linux \"perf\" profiler with DWARF support;\n\ - also PYTHON_PERF_JIT_SUPPORT=1\n\ -" +"#s{-X} #L{importtime}#b{[=2]}: show how long each import takes; use #S{-X} #e{importtime}#B{=2} to\n" +" log imports of already-loaded modules; also #e{PYTHONPROFILEIMPORTTIME}\n" +"#s{-X} #L{int_max_str_digits}#b{=N}: limit the size of int<->str conversions;\n" +" 0 disables the limit; also #e{PYTHONINTMAXSTRDIGITS}\n" +"#s{-X} #L{lazy_imports}#b{=[all|normal]}: control global lazy imports;\n" +" default is #B{normal}; also #e{PYTHON_LAZY_IMPORTS}\n" +"#s{-X} #L{no_debug_ranges}: don't include extra location information in code objects;\n" +" also #e{PYTHONNODEBUGRANGES}\n" +"#s{-X} #L{pathconfig_warnings}#b{=[0|1]}: if true (#B{1}) then path configuration is allowed\n" +" to log warnings into stderr; if false (#B{0}) suppress these warnings;\n" +" set to true by default; also #e{PYTHON_PATHCONFIG_WARNINGS}\n" +"#s{-X} #L{perf}: support the Linux \"perf\" profiler; also #e{PYTHONPERFSUPPORT}#B{=1}\n" +"#s{-X} #L{perf_jit}: support the Linux \"perf\" profiler with DWARF support;\n" +" also #e{PYTHON_PERF_JIT_SUPPORT}#B{=1}\n" #ifdef Py_DEBUG -"-X presite=MOD: import this module before site; also PYTHON_PRESITE\n" +"#s{-X} #L{presite}#b{=MOD}: import this module before site; also #e{PYTHON_PRESITE}\n" #endif -"\ --X pycache_prefix=PATH: write .pyc files to a parallel tree instead of to the\n\ - code tree; also PYTHONPYCACHEPREFIX\n\ -" +"#s{-X} #L{pycache_prefix}#b{=PATH}: write .pyc files to a parallel tree instead of to the\n" +" code tree; also #e{PYTHONPYCACHEPREFIX}\n" #ifdef Py_STATS -"-X pystats: enable pystats collection at startup; also PYTHONSTATS\n" +"#s{-X} #L{pystats}: enable pystats collection at startup; also #e{PYTHONSTATS}\n" #endif -"\ --X showrefcount: output the total reference count and number of used\n\ - memory blocks when the program finishes or after each statement in\n\ - the interactive interpreter; only works on debug builds\n\ --X thread_inherit_context=[0|1]: enable (1) or disable (0) threads inheriting\n\ - context vars by default; enabled by default in the free-threaded\n\ - build and disabled otherwise; also PYTHON_THREAD_INHERIT_CONTEXT\n\ -" +"#s{-X} #L{showrefcount}: output the total reference count and number of used\n" +" memory blocks when the program finishes or after each statement in\n" +" the interactive interpreter; only works on debug builds\n" +"#s{-X} #L{thread_inherit_context}#b{=[0|1]}: enable (#B{1}) or disable (#B{0}) threads inheriting\n" +" context vars by default; enabled by default in the free-threaded\n" +" build and disabled otherwise; also #e{PYTHON_THREAD_INHERIT_CONTEXT}\n" #ifdef Py_GIL_DISABLED -"-X tlbc=[0|1]: enable (1) or disable (0) thread-local bytecode. Also\n\ - PYTHON_TLBC\n" +"#s{-X} #L{tlbc}#b{=[0|1]}: enable (#B{1}) or disable (#B{0}) thread-local bytecode. Also\n" +" #e{PYTHON_TLBC}\n" #endif -"\ --X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n\ - of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\ --X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\ --X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None';\n\ - also PYTHONWARNDEFAULTENCODING\ -"; +"#s{-X} #L{tracemalloc}#b{[=N]}: trace Python memory allocations; N sets a traceback limit\n" +" of #B{N} frames (default: #B{1}); also #e{PYTHONTRACEMALLOC}#B{=N}\n" +"#s{-X} #L{utf8}#b{[=0|1]}: enable (#B{1}) or disable (#B{0}) UTF-8 mode; also #e{PYTHONUTF8}\n" +"#s{-X} #L{warn_default_encoding}: enable opt-in EncodingWarning for 'encoding=None';\n" +" also #e{PYTHONWARNDEFAULTENCODING}\n" +; /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = -"Environment variables that change behavior:\n" -"PYTHONASYNCIODEBUG: enable asyncio debug mode\n" -"PYTHON_BASIC_REPL: use the traditional parser-based REPL\n" -"PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" +"#h{Environment variables that change behavior:}\n" +"#E{PYTHONASYNCIODEBUG}: enable asyncio debug mode\n" +"#E{PYTHON_BASIC_REPL}: use the traditional parser-based REPL\n" +"#E{PYTHONBREAKPOINT}: if this variable is set to #B{0}, it disables the default\n" " debugger. It can be set to the callable of your debugger of\n" " choice.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows)\n" -"PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" -" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request\n" +"#E{PYTHONCASEOK} : ignore case in 'import' statements (Windows)\n" +"#E{PYTHONCOERCECLOCALE}: if this variable is set to #B{0}, it disables the locale\n" +" coercion behavior. Use #e{PYTHONCOERCECLOCALE}#B{=warn} to request\n" " display of locale coercion and locale compatibility warnings\n" " on stderr.\n" -"PYTHON_COLORS : if this variable is set to 1, the interpreter will colorize\n" -" various kinds of output. Setting it to 0 deactivates\n" +"#E{PYTHON_COLORS} : if this variable is set to #B{1}, the interpreter will colorize\n" +" various kinds of output. Setting it to #B{0} deactivates\n" " this behavior.\n" #ifdef Py_TRACE_REFS -"PYTHONDUMPREFS : dump objects and reference counts still alive after shutdown\n" -"PYTHONDUMPREFSFILE: dump objects and reference counts to the specified file\n" +"#E{PYTHONDUMPREFS} : dump objects and reference counts still alive after shutdown\n" +"#E{PYTHONDUMPREFSFILE}: dump objects and reference counts to the specified file\n" #endif #ifdef __APPLE__ -"PYTHONEXECUTABLE: set sys.argv[0] to this value (macOS only)\n" +"#E{PYTHONEXECUTABLE}: set sys.argv[0] to this value (macOS only)\n" #endif -"PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" +"#E{PYTHONHASHSEED} : if this variable is set to 'random', a random value is used\n" " to seed the hashes of str and bytes objects. It can also be\n" " set to an integer in the range [0,4294967295] to get hash\n" " values with a predictable seed.\n" -"PYTHON_HISTORY : the location of a .python_history file.\n" -"PYTHONHOME : alternate <prefix> directory (or <prefix>%lc<exec_prefix>).\n" -" The default module search path uses %s.\n" -"PYTHONIOENCODING: encoding[:errors] used for stdin/stdout/stderr\n" +"#E{PYTHON_HISTORY} : the location of a .python_history file.\n" +"#E{PYTHONHOME} : alternate <prefix> directory (or <prefix>#D<exec_prefix>).\n" +" The default module search path uses #H.\n" +"#E{PYTHONIOENCODING}: encoding[:errors] used for stdin/stdout/stderr\n" #ifdef MS_WINDOWS -"PYTHONLEGACYWINDOWSFSENCODING: use legacy \"mbcs\" encoding for file system\n" -"PYTHONLEGACYWINDOWSSTDIO: use legacy Windows stdio\n" +"#E{PYTHONLEGACYWINDOWSFSENCODING}: use legacy \"mbcs\" encoding for file system\n" +"#E{PYTHONLEGACYWINDOWSSTDIO}: use legacy Windows stdio\n" #endif -"PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" -" on Python memory allocators. Use PYTHONMALLOC=debug to\n" +"#E{PYTHONMALLOC} : set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use #e{PYTHONMALLOC}#B{=debug} to\n" " install debug hooks.\n" -"PYTHONMALLOCSTATS: print memory allocator statistics\n" -"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" +"#E{PYTHONMALLOCSTATS}: print memory allocator statistics\n" +"#E{PYTHONPATH} : '#D'-separated list of directories prefixed to the\n" " default module search path. The result is sys.path.\n" -"PYTHONPLATLIBDIR: override sys.platlibdir\n" -"PYTHONSTARTUP : file executed on interactive startup (no default)\n" -"PYTHONUSERBASE : defines the user base directory (site.USER_BASE)\n" +"#E{PYTHONPLATLIBDIR}: override sys.platlibdir\n" +"#E{PYTHONSTARTUP} : file executed on interactive startup (no default)\n" +"#E{PYTHONUSERBASE} : defines the user base directory (site.USER_BASE)\n" "\n" -"These variables have equivalent command-line options (see --help for details):\n" -"PYTHON_CONTEXT_AWARE_WARNINGS: if true (1), enable thread-safe warnings\n" -" module behaviour (-X context_aware_warnings)\n" -"PYTHON_CPU_COUNT: override the return value of os.cpu_count() (-X cpu_count)\n" -"PYTHONDEBUG : enable parser debug mode (-d)\n" -"PYTHONDEVMODE : enable Python Development Mode (-X dev)\n" -"PYTHONDONTWRITEBYTECODE: don't write .pyc files (-B)\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors (-X faulthandler)\n" -"PYTHON_FROZEN_MODULES: whether to use frozen modules; the default is \"on\"\n" -" for installed Python and \"off\" for a local build\n" -" (-X frozen_modules)\n" +"#h{These variables have equivalent command-line options (see }#e{--help} for details):\n" +"#E{PYTHON_CONTEXT_AWARE_WARNINGS}: if true (#B{1}), enable thread-safe warnings\n" +" module behaviour (#S{-X} #e{context_aware_warnings})\n" +"#E{PYTHON_CPU_COUNT}: override the return value of os.cpu_count() (#S{-X} #e{cpu_count})\n" +"#E{PYTHONDEBUG} : enable parser debug mode (#S{-d})\n" +"#E{PYTHONDEVMODE} : enable Python Development Mode (#S{-X} #e{dev})\n" +"#E{PYTHONDONTWRITEBYTECODE}: don't write .pyc files (#S{-B})\n" +"#E{PYTHONFAULTHANDLER}: dump the Python traceback on fatal errors (#S{-X} #e{faulthandler})\n" +"#E{PYTHON_FROZEN_MODULES}: whether to use frozen modules; the default is \"#B{on}\"\n" +" for installed Python and \"#B{off}\" for a local build\n" +" (#S{-X} #e{frozen_modules})\n" #ifdef Py_GIL_DISABLED -"PYTHON_GIL : when set to 0, disables the GIL (-X gil)\n" +"#E{PYTHON_GIL} : when set to #B{0}, disables the GIL (#S{-X} #e{gil})\n" #endif -"PYTHONINSPECT : inspect interactively after running script (-i)\n" -"PYTHONINTMAXSTRDIGITS: limit the size of int<->str conversions;\n" -" 0 disables the limit (-X int_max_str_digits=N)\n" -"PYTHON_LAZY_IMPORTS: control global lazy imports (-X lazy_imports)\n" -"PYTHONNODEBUGRANGES: don't include extra location information in code objects\n" -" (-X no_debug_ranges)\n" -"PYTHONNOUSERSITE: disable user site directory (-s)\n" -"PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" -"PYTHON_PERF_JIT_SUPPORT: enable Linux \"perf\" profiler support with JIT\n" -" (-X perf_jit)\n" -"PYTHONPERFSUPPORT: support the Linux \"perf\" profiler (-X perf)\n" +"#E{PYTHONINSPECT} : inspect interactively after running script (#S{-i})\n" +"#E{PYTHONINTMAXSTRDIGITS}: limit the size of int<->str conversions;\n" +" 0 disables the limit (#S{-X} #e{int_max_str_digits}#B{=N})\n" +"#E{PYTHON_LAZY_IMPORTS}: control global lazy imports (#S{-X} #e{lazy_imports})\n" +"#E{PYTHONNODEBUGRANGES}: don't include extra location information in code objects\n" +" (#S{-X} #e{no_debug_ranges})\n" +"#E{PYTHONNOUSERSITE}: disable user site directory (#S{-s})\n" +"#E{PYTHONOPTIMIZE} : enable level 1 optimizations (#S{-O})\n" +"#E{PYTHON_PERF_JIT_SUPPORT}: enable Linux \"perf\" profiler support with JIT\n" +" (#S{-X} #e{perf_jit})\n" +"#E{PYTHONPERFSUPPORT}: support the Linux \"perf\" profiler (#S{-X} #e{perf})\n" #ifdef Py_DEBUG -"PYTHON_PRESITE: import this module before site (-X presite)\n" +"#E{PYTHON_PRESITE}: import this module before site (#S{-X} #e{presite})\n" #endif -"PYTHONPROFILEIMPORTTIME: show how long each import takes (-X importtime)\n" -"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files\n" -" (-X pycache_prefix)\n" -"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" +"#E{PYTHONPROFILEIMPORTTIME}: show how long each import takes (#S{-X} #e{importtime})\n" +"#E{PYTHONPYCACHEPREFIX}: root directory for bytecode cache (pyc) files\n" +" (#S{-X} #e{pycache_prefix})\n" +"#E{PYTHONSAFEPATH} : don't prepend a potentially unsafe path to sys.path.\n" #ifdef Py_STATS -"PYTHONSTATS : turns on statistics gathering (-X pystats)\n" +"#E{PYTHONSTATS} : turns on statistics gathering (#S{-X} #e{pystats})\n" #endif -"PYTHON_THREAD_INHERIT_CONTEXT: if true (1), threads inherit context vars\n" -" (-X thread_inherit_context)\n" +"#E{PYTHON_THREAD_INHERIT_CONTEXT}: if true (#B{1}), threads inherit context vars\n" +" (#S{-X} #e{thread_inherit_context})\n" #ifdef Py_GIL_DISABLED -"PYTHON_TLBC : when set to 0, disables thread-local bytecode (-X tlbc)\n" +"#E{PYTHON_TLBC} : when set to #B{0}, disables thread-local bytecode (#S{-X} #e{tlbc})\n" #endif -"PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n" -"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" -"PYTHONUTF8 : control the UTF-8 mode (-X utf8)\n" -"PYTHONVERBOSE : trace import statements (-v)\n" -"PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'\n" -" (-X warn_default_encoding)\n" -"PYTHONWARNINGS : warning control (-W)\n" +"#E{PYTHONTRACEMALLOC}: trace Python memory allocations (#S{-X} #e{tracemalloc})\n" +"#E{PYTHONUNBUFFERED}: disable stdout/stderr buffering (#S{-u})\n" +"#E{PYTHONUTF8} : control the UTF-8 mode (#S{-X} #e{utf8})\n" +"#E{PYTHONVERBOSE} : trace import statements (#S{-v})\n" +"#E{PYTHONWARNDEFAULTENCODING}: enable opt-in EncodingWarning for 'encoding=None'\n" +" (#S{-X} #e{warn_default_encoding})\n" +"#E{PYTHONWARNINGS} : warning control (#S{-W})\n" ; -#if defined(MS_WINDOWS) -# define PYTHONHOMEHELP "<prefix>\\python{major}{minor}" -#else -# define PYTHONHOMEHELP "<prefix>/lib/pythonX.X" -#endif - /* --- Global configuration variables ----------------------------- */ @@ -949,7 +1065,8 @@ config_check_consistency(const PyConfig *config) assert(config->int_max_str_digits >= 0); // cpu_count can be -1 if the user doesn't override it. assert(config->cpu_count != 0); - // lazy_imports can be -1 (default), 0 (off), or 1 (on). + // lazy_imports can be -1 (default) or 1 (on). 0 is rejected later + // for embedders with an error message. assert(config->lazy_imports >= -1 && config->lazy_imports <= 1); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). @@ -2321,15 +2438,12 @@ config_init_lazy_imports(PyConfig *config) if (strcmp(env, "all") == 0) { lazy_imports = 1; } - else if (strcmp(env, "none") == 0) { - lazy_imports = 0; - } else if (strcmp(env, "normal") == 0) { lazy_imports = -1; } else { return _PyStatus_ERR("PYTHON_LAZY_IMPORTS: invalid value; " - "expected 'all', 'none', or 'normal'"); + "expected 'all' or 'normal'"); } config->lazy_imports = lazy_imports; } @@ -2339,15 +2453,12 @@ config_init_lazy_imports(PyConfig *config) if (wcscmp(x_value, L"all") == 0) { lazy_imports = 1; } - else if (wcscmp(x_value, L"none") == 0) { - lazy_imports = 0; - } else if (wcscmp(x_value, L"normal") == 0) { lazy_imports = -1; } else { return _PyStatus_ERR("-X lazy_imports: invalid value; " - "expected 'all', 'none', or 'normal'"); + "expected 'all' or 'normal'"); } config->lazy_imports = lazy_imports; } @@ -2935,25 +3046,29 @@ static void config_usage(int error, const wchar_t* program) { FILE *f = error ? stderr : stdout; + int colorize = _Py_can_colorize(f); - fprintf(f, usage_line, program); - if (error) + fprint_help(f, usage_line, colorize, program); + if (error) { fprintf(f, "Try `python -h' for more information.\n"); + } else { - fputs(usage_help, f); + fprint_help(f, usage_help, colorize, NULL); } } static void config_envvars_usage(void) { - printf(usage_envvars, (wint_t)DELIM, PYTHONHOMEHELP, (wint_t)DELIM); + int colorize = _Py_can_colorize(stdout); + fprint_help(stdout, usage_envvars, colorize, NULL); } static void config_xoptions_usage(void) { - puts(usage_xoptions); + int colorize = _Py_can_colorize(stdout); + fprint_help(stdout, usage_xoptions, colorize, NULL); } static void @@ -3568,6 +3683,44 @@ PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, } +#ifdef __CYGWIN__ +// Cygwin strips ".exe" suffix from argv[0]. +// Add again the ".exe" suffix. +static PyStatus +config_argv0_add_exe(PyConfig *config) +{ + if (config->argv.length < 1) { + return _PyStatus_OK(); + } + const wchar_t *argv0 = config->argv.items[0]; + size_t len = wcslen(argv0); + if (len >= 5 && wcscmp(argv0 + len - 4, L".exe") == 0) { + return _PyStatus_OK(); + } + + wchar_t *exe = PyMem_RawMalloc((len + 4 + 1) * sizeof(wchar_t)); + if (exe == NULL) { + return _PyStatus_NO_MEMORY(); + } + wcscpy(exe, argv0); + wcscat(exe, L".exe"); + + FILE *fp = _Py_wfopen(exe, L"rb"); + if (fp != NULL) { + fclose(fp); + + PyMem_RawFree(config->argv.items[0]); + config->argv.items[0] = exe; + } + else { + PyMem_RawFree(exe); + } + + return _PyStatus_OK(); +} +#endif + + /* Read the configuration into PyConfig from: * Command line arguments @@ -3587,6 +3740,13 @@ _PyConfig_Read(PyConfig *config, int compute_path_config) config_get_global_vars(config); +#ifdef __CYGWIN__ + status = config_argv0_add_exe(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + if (config->orig_argv.length == 0 && !(config->argv.length == 1 && wcscmp(config->argv.items[0], L"") == 0)) @@ -4481,7 +4641,8 @@ config_get(const PyConfig *config, const PyConfigSpec *spec, if (strcmp(spec->name, "int_max_str_digits") == 0) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->long_state.max_str_digits); + int maxdigits = _Py_atomic_load_int(&interp->long_state.max_str_digits); + return PyLong_FromLong(maxdigits); } } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 51bcbfdb3b6c55b..0af2070b5cd983a 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -2630,8 +2630,9 @@ capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t PyErr_SetString(PyExc_ValueError, "offset must be non-negative"); return -1; } + PyObject *offset_obj = NULL; if (event != PY_MONITORING_EVENT_LINE) { - PyObject *offset_obj = PyLong_FromLong(offset); + offset_obj = PyLong_FromLong(offset); if (offset_obj == NULL) { return -1; } @@ -2672,6 +2673,7 @@ capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t } } } + Py_XDECREF(offset_obj); return err; } diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 9cfc285c6a59259..f081f33cc83b88c 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -7,6 +7,9 @@ #include "pycore_genobject.h" // _PyAsyncGenValueWrapperNew #include "pycore_interpframe.h" // _PyFrame_GetLocals() #include "pycore_intrinsics.h" // INTRINSIC_PRINT +#include "pycore_list.h" // _PyList_AsTupleAndClear() +#include "pycore_object.h" // _PyObject_IsUniquelyReferenced() +#include "pycore_setobject.h" // _PySet_Freeze() #include "pycore_pyerrors.h" // _PyErr_SetString() #include "pycore_runtime.h" // _Py_ID() #include "pycore_typevarobject.h" // _Py_make_typevar() @@ -190,8 +193,12 @@ unary_pos(PyThreadState* unused, PyObject *value) static PyObject * list_to_tuple(PyThreadState* unused, PyObject *v) { - assert(PyList_Check(v)); - return PyTuple_FromArray(((PyListObject *)v)->ob_item, Py_SIZE(v)); + /* INTRINSIC_LIST_TO_TUPLE is only emitted by the compiler for a + freshly-built, uniquely-referenced temporary list, so steal its items + into the tuple instead of copying them. */ + assert(PyList_CheckExact(v)); + assert(_PyObject_IsUniquelyReferenced(v)); + return _PyList_AsTupleAndClear((PyListObject *)v); } static PyObject * @@ -201,6 +208,14 @@ make_typevar(PyThreadState* Py_UNUSED(ignored), PyObject *v) return _Py_make_typevar(v, NULL, NULL); } +static PyObject * +make_frozenset(PyThreadState* Py_UNUSED(ignored), PyObject *set) +{ + assert(PySet_CheckExact(set)); + assert(_PyObject_IsUniquelyReferenced(set)); + return _PySet_Freeze(set); +} + #define INTRINSIC_FUNC_ENTRY(N, F) \ [N] = {F, #N}, @@ -219,6 +234,7 @@ _PyIntrinsics_UnaryFunctions[] = { INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVARTUPLE, _Py_make_typevartuple) INTRINSIC_FUNC_ENTRY(INTRINSIC_SUBSCRIPT_GENERIC, _Py_subscript_generic) INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEALIAS, _Py_make_typealias) + INTRINSIC_FUNC_ENTRY(INTRINSIC_BUILD_FROZENSET, make_frozenset) }; diff --git a/Python/jit.c b/Python/jit.c index 8b555105129a9f4..50e606a7a0963f6 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -11,11 +11,12 @@ #include "pycore_floatobject.h" #include "pycore_frame.h" #include "pycore_function.h" +#include "pycore_genobject.h" #include "pycore_import.h" #include "pycore_interpframe.h" #include "pycore_interpolation.h" #include "pycore_intrinsics.h" -#include "pycore_jit_unwind.h" +#include "pycore_jit_publish.h" #include "pycore_lazyimportobject.h" #include "pycore_list.h" #include "pycore_long.h" @@ -32,7 +33,7 @@ #include "pycore_jit.h" -// Memory management stuff: //////////////////////////////////////////////////// +// Memory management stuff: /////////////////////////////////////////////////// #ifndef MS_WINDOWS #include <sys/mman.h> @@ -61,40 +62,6 @@ jit_error(const char *message) PyErr_Format(PyExc_RuntimeWarning, "JIT %s (%d)", message, hint); } -/* - * Publish JIT code to optional tooling backends. - * - * The return value is a backend-specific deregistration handle, not a - * success/failure indicator. NULL means there is nothing to unregister later: - * perf does not need a handle, and GDB registration failures are intentionally - * non-fatal because tooling support must not make JIT compilation fail. - */ -static void * -jit_record_code(const void *code_addr, size_t code_size, - const char *entry, const char *filename) -{ -#ifdef PY_HAVE_PERF_TRAMPOLINE - _PyPerf_Callbacks callbacks; - _PyPerfTrampoline_GetCallbacks(&callbacks); - if (callbacks.write_state == _Py_perfmap_jit_callbacks.write_state) { - _PyPerfJit_WriteNamedCode( - code_addr, code_size, entry, filename); - return NULL; - } -#endif - -#if defined(PY_HAVE_JIT_GDB_UNWIND) - return _PyJitUnwind_GdbRegisterCode( - code_addr, code_size, entry, filename); -#else - (void)code_addr; - (void)code_size; - (void)entry; - (void)filename; - return NULL; -#endif -} - static int address_in_executor_array(_PyExecutorObject **ptrs, size_t count, uintptr_t addr) { @@ -750,10 +717,11 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz } executor->jit_code = memory; executor->jit_size = total_size; - executor->jit_gdb_handle = jit_record_code(memory, - code_size + state.trampolines.size, - "jit", - "executor"); + executor->jit_registration = _PyJit_RegisterCode( + memory, + code_size + state.trampolines.size, + "jit", + "executor"); return 0; } @@ -766,12 +734,8 @@ _PyJIT_Free(_PyExecutorObject *executor) if (memory) { executor->jit_code = NULL; executor->jit_size = 0; -#if defined(PY_HAVE_JIT_GDB_UNWIND) - if (executor->jit_gdb_handle != NULL) { - _PyJitUnwind_GdbUnregisterCode(executor->jit_gdb_handle); - executor->jit_gdb_handle = NULL; - } -#endif + _PyJit_UnregisterCode(executor->jit_registration); + executor->jit_registration = NULL; if (jit_free(memory, size)) { PyErr_FormatUnraisable("Exception ignored while " "freeing JIT memory"); @@ -779,4 +743,14 @@ _PyJIT_Free(_PyExecutorObject *executor) } } +// Avoid excessive bloat due to asserts in stencils +int +_Py_jit_assertion_failure(int line) +{ + printf("Assertion failure at line %d of executor_cases.c.h", line); + fflush(stdout); + abort(); + return 0; +} + #endif // _Py_JIT diff --git a/Python/jit_publish.c b/Python/jit_publish.c new file mode 100644 index 000000000000000..fd068076a7709ea --- /dev/null +++ b/Python/jit_publish.c @@ -0,0 +1,137 @@ +#include "Python.h" + +#include "pycore_ceval.h" +#include "pycore_jit_publish.h" +#include "pycore_jit_unwind.h" + +#ifdef _Py_JIT + +#if defined(PY_HAVE_JIT_GDB_UNWIND) \ + || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) +struct _PyJitCodeRegistration { + void *gdb_handle; + void *gnu_backtrace_handle; +}; +#endif + +static void +jit_register_perf_code(const void *code_addr, size_t code_size, + const char *entry, const char *filename) +{ +#ifdef PY_HAVE_PERF_TRAMPOLINE + _PyPerf_Callbacks callbacks; + _PyPerfTrampoline_GetCallbacks(&callbacks); + if (callbacks.write_state == _Py_perfmap_jit_callbacks.write_state) { + _PyPerfJit_WriteNamedCode( + code_addr, code_size, entry, filename); + } +#else + (void)code_addr; + (void)code_size; + (void)entry; + (void)filename; +#endif +} + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +static void +jit_register_gdb_code(_PyJitCodeRegistration *registration, + const void *code_addr, size_t code_size, + const char *entry, const char *filename) +{ + registration->gdb_handle = _PyJitUnwind_GdbRegisterCode( + code_addr, code_size, entry, filename); +} + +static void +jit_unregister_gdb_code(_PyJitCodeRegistration *registration) +{ + if (registration->gdb_handle != NULL) { + _PyJitUnwind_GdbUnregisterCode(registration->gdb_handle); + registration->gdb_handle = NULL; + } +} +#endif + +#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) +static void +jit_register_gnu_backtrace_code(_PyJitCodeRegistration *registration, + const void *code_addr, size_t code_size) +{ + registration->gnu_backtrace_handle = + _PyJitUnwind_GnuBacktraceRegisterCode(code_addr, code_size); +} + +static void +jit_unregister_gnu_backtrace_code(_PyJitCodeRegistration *registration) +{ + if (registration->gnu_backtrace_handle != NULL) { + _PyJitUnwind_GnuBacktraceUnregisterCode( + registration->gnu_backtrace_handle); + registration->gnu_backtrace_handle = NULL; + } +} +#endif + +_PyJitCodeRegistration * +_PyJit_RegisterCode(const void *code_addr, size_t code_size, + const char *entry, const char *filename) +{ + jit_register_perf_code(code_addr, code_size, entry, filename); + // Perf publication has no teardown handle, so it is intentionally + // not counted below. + +#if !defined(PY_HAVE_JIT_GDB_UNWIND) \ + && !defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) + return NULL; +#else + _PyJitCodeRegistration *registration = PyMem_RawCalloc( + 1, sizeof(*registration)); + if (registration == NULL) { + return NULL; + } + + // Partial failures are non-fatal: the JIT code can still execute, but + // unavailable tooling may not be able to unwind it. + int any_registered = 0; +# if defined(PY_HAVE_JIT_GDB_UNWIND) + jit_register_gdb_code( + registration, code_addr, code_size, entry, filename); + any_registered |= registration->gdb_handle != NULL; +# endif +# if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) + jit_register_gnu_backtrace_code( + registration, code_addr, code_size); + any_registered |= registration->gnu_backtrace_handle != NULL; +# endif + if (!any_registered) { + PyMem_RawFree(registration); + return NULL; + } + return registration; +#endif +} + +void +_PyJit_UnregisterCode(_PyJitCodeRegistration *registration) +{ +#if !defined(PY_HAVE_JIT_GDB_UNWIND) \ + && !defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) + assert(registration == NULL); + (void)registration; +#else + if (registration == NULL) { + return; + } + +# if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) + jit_unregister_gnu_backtrace_code(registration); +# endif +# if defined(PY_HAVE_JIT_GDB_UNWIND) + jit_unregister_gdb_code(registration); +# endif + PyMem_RawFree(registration); +#endif +} + +#endif // _Py_JIT diff --git a/Python/jit_unwind.c b/Python/jit_unwind.c index 09002feafec17ca..0941ed593ff7d14 100644 --- a/Python/jit_unwind.c +++ b/Python/jit_unwind.c @@ -16,11 +16,22 @@ # endif #endif -#if defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) +#if defined(PY_HAVE_PERF_TRAMPOLINE) \ + || defined(PY_HAVE_JIT_GDB_UNWIND) \ + || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) #if defined(PY_HAVE_JIT_GDB_UNWIND) # include <elf.h> #endif +#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) +/* + * libgcc exposes frame registration entry points, but GCC's public headers + * on some distributions do not declare them even though the symbols are + * available in libgcc_s. + */ +void __register_frame(const void *); +void __deregister_frame(const void *); +#endif #include <stdio.h> #include <string.h> @@ -49,6 +60,9 @@ enum { DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter DWRF_CFA_offset = 0x80, // Simple offset instruction +#if defined(__aarch64__) + DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state +#endif DWRF_CFA_restore = 0xc0 // Restore register }; @@ -551,6 +565,13 @@ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 calling convention unwinding rules */ +#if defined(__ARM_FEATURE_PAC_DEFAULT) || \ + (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) +#endif +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here +#endif DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 DWRF_UV(16); // Stack pointer moved by 16 bytes @@ -559,6 +580,9 @@ static void elf_init_ehframe_perf(ELFObjectContext* ctx) { DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed +#endif DWRF_U8(DWRF_CFA_def_cfa_register); // CFA = FP (x29) + 16 DWRF_UV(DWRF_REG_FP); DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! @@ -983,4 +1007,56 @@ _PyJitUnwind_GdbUnregisterCode(void *handle) #endif } -#endif // defined(PY_HAVE_PERF_TRAMPOLINE) || defined(PY_HAVE_JIT_GDB_UNWIND) +#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) +void * +_PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr, size_t code_size) +{ + if (code_addr == NULL || code_size == 0) { + return NULL; + } + + size_t eh_frame_size = _PyJitUnwind_EhFrameSize(1); + if (eh_frame_size == 0) { + return NULL; + } + size_t total_size = eh_frame_size + sizeof(uint32_t); + if (total_size < eh_frame_size) { + return NULL; + } + + /* + * libgcc's __register_frame walks a .eh_frame section until it finds a + * zero-length terminator entry, so keep an extra zeroed word after the + * generated CIE/FDE pair. + * + * See GCC's libgcc/unwind-dw2-fde.c (__register_frame) and + * libgcc/unwind-dw2-fde.h (last_fde/next_fde): + * https://github.com/gcc-mirror/gcc/blob/master/libgcc/unwind-dw2-fde.c + * https://github.com/gcc-mirror/gcc/blob/master/libgcc/unwind-dw2-fde.h + */ + uint8_t *eh_frame = PyMem_RawCalloc(1, total_size); + if (eh_frame == NULL) { + return NULL; + } + if (_PyJitUnwind_BuildEhFrame( + eh_frame, eh_frame_size, code_addr, code_size, 1) == 0) { + PyMem_RawFree(eh_frame); + return NULL; + } + + __register_frame(eh_frame); + return eh_frame; +} + +void +_PyJitUnwind_GnuBacktraceUnregisterCode(void *handle) +{ + if (handle == NULL) { + return; + } + __deregister_frame(handle); + PyMem_RawFree(handle); +} +#endif // defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) + +#endif // JIT unwind support diff --git a/Python/marshal.c b/Python/marshal.c index 990afefe0d3b419..9688d426419c2fa 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -2061,7 +2061,6 @@ marshal_load_impl(PyObject *module, PyObject *file, int allow_code) /*[clinic input] @permit_long_summary -@permit_long_docstring_body marshal.dumps value: object @@ -2075,14 +2074,14 @@ marshal.dumps Return the bytes object that would be written to a file by dump(value, file). -Raise a ValueError exception if value has (or contains an object that has) an -unsupported type. +Raise a ValueError exception if value has (or contains an object that +has) an unsupported type. [clinic start generated code]*/ static PyObject * marshal_dumps_impl(PyObject *module, PyObject *value, int version, int allow_code) -/*[clinic end generated code: output=115f90da518d1d49 input=80cd3f30c1637ade]*/ +/*[clinic end generated code: output=115f90da518d1d49 input=dc1edcafd43124c5]*/ { return _PyMarshal_WriteObjectToString(value, version, allow_code); } @@ -2098,13 +2097,13 @@ marshal.loads Convert the bytes-like object to a value. -If no valid value is found, raise EOFError, ValueError or TypeError. Extra -bytes in the input are ignored. +If no valid value is found, raise EOFError, ValueError or TypeError. +Extra bytes in the input are ignored. [clinic start generated code]*/ static PyObject * marshal_loads_impl(PyObject *module, Py_buffer *bytes, int allow_code) -/*[clinic end generated code: output=62c0c538d3edc31f input=14de68965b45aaa7]*/ +/*[clinic end generated code: output=62c0c538d3edc31f input=286f1dbd6811d2ad]*/ { RFILE rf; char *s = bytes->buf; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index d99b618c8393fcb..1a7eb9169fc837c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -202,7 +202,9 @@ static void *opcode_targets_table[256] = { &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_RESUME_CHECK, &&TARGET_RESUME_CHECK_JIT, + &&TARGET_SEND_ASYNC_GEN, &&TARGET_SEND_GEN, + &&TARGET_SEND_VIRTUAL, &&TARGET_STORE_ATTR_INSTANCE_VALUE, &&TARGET_STORE_ATTR_SLOT, &&TARGET_STORE_ATTR_WITH_HINT, @@ -231,8 +233,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_POP_ITER, &&TARGET_INSTRUMENTED_END_SEND, @@ -476,8 +476,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, &&TARGET_TRACE_RECORD, - &&_unknown_opcode, - &&_unknown_opcode, + &&TARGET_TRACE_RECORD, + &&TARGET_TRACE_RECORD, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -527,6 +527,7 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_error(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_exit_unwind_notrace(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_start_frame(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); @@ -724,7 +725,9 @@ static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RESUME_CHECK_JIT(TAIL_CALL_PARAM static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_ASYNC_GEN(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SEND_VIRTUAL(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); static PyObject *Py_PRESERVE_NONE_CC _TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); @@ -968,7 +971,9 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [RETURN_GENERATOR] = _TAIL_CALL_RETURN_GENERATOR, [RETURN_VALUE] = _TAIL_CALL_RETURN_VALUE, [SEND] = _TAIL_CALL_SEND, + [SEND_ASYNC_GEN] = _TAIL_CALL_SEND_ASYNC_GEN, [SEND_GEN] = _TAIL_CALL_SEND_GEN, + [SEND_VIRTUAL] = _TAIL_CALL_SEND_VIRTUAL, [SETUP_ANNOTATIONS] = _TAIL_CALL_SETUP_ANNOTATIONS, [SET_ADD] = _TAIL_CALL_SET_ADD, [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_SET_FUNCTION_ATTRIBUTE, @@ -1014,8 +1019,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [217] = _TAIL_CALL_UNKNOWN_OPCODE, - [218] = _TAIL_CALL_UNKNOWN_OPCODE, [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1226,7 +1229,9 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [RETURN_GENERATOR] = _TAIL_CALL_TRACE_RECORD, [RETURN_VALUE] = _TAIL_CALL_TRACE_RECORD, [SEND] = _TAIL_CALL_TRACE_RECORD, + [SEND_ASYNC_GEN] = _TAIL_CALL_TRACE_RECORD, [SEND_GEN] = _TAIL_CALL_TRACE_RECORD, + [SEND_VIRTUAL] = _TAIL_CALL_TRACE_RECORD, [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACE_RECORD, [SET_ADD] = _TAIL_CALL_TRACE_RECORD, [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACE_RECORD, @@ -1272,8 +1277,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [125] = _TAIL_CALL_UNKNOWN_OPCODE, [126] = _TAIL_CALL_UNKNOWN_OPCODE, [127] = _TAIL_CALL_UNKNOWN_OPCODE, - [217] = _TAIL_CALL_UNKNOWN_OPCODE, - [218] = _TAIL_CALL_UNKNOWN_OPCODE, [219] = _TAIL_CALL_UNKNOWN_OPCODE, [220] = _TAIL_CALL_UNKNOWN_OPCODE, [221] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index 11658fca0da5954..e95e4b5e24b2c54 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -500,6 +500,8 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO, [_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK, [_LOAD_BYTECODE] = _NOP, + [_SEND_VIRTUAL] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_ASYNC_GEN] = _SEND_ASYNC_GEN_TIER_TWO, }; static const uint8_t @@ -508,6 +510,7 @@ is_for_iter_test[MAX_UOP_ID + 1] = { [_GUARD_NOT_EXHAUSTED_LIST] = 1, [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, [_FOR_ITER_TIER_TWO] = 1, + [_ITER_NEXT_INLINE] = 1, }; static const uint16_t @@ -660,44 +663,6 @@ is_terminator(const _PyUOpInstruction *uop) ); } -static PyObject * -record_trace_transform_to_type(PyObject *value) -{ - PyObject *tp = Py_NewRef((PyObject *)Py_TYPE(value)); - Py_DECREF(value); - return tp; -} - -/* _RECORD_NOS_GEN_FUNC and _RECORD_3OS_GEN_FUNC record the raw receiver. - * If it is a generator, return its function object; otherwise return NULL. - */ -static PyObject * -record_trace_transform_gen_func(PyObject *value) -{ - PyObject *func = NULL; - if (PyGen_Check(value)) { - _PyStackRef f = ((PyGenObject *)value)->gi_iframe.f_funcobj; - if (!PyStackRef_IsNull(f)) { - func = Py_NewRef(PyStackRef_AsPyObjectBorrow(f)); - } - } - Py_DECREF(value); - return func; -} - -/* _RECORD_BOUND_METHOD records the raw callable. - * Keep it only for bound methods; otherwise return NULL. - */ -static PyObject * -record_trace_transform_bound_method(PyObject *value) -{ - if (Py_TYPE(value) == &PyMethod_Type) { - return value; - } - Py_DECREF(value); - return NULL; -} - /* Returns 1 on success (added to trace), 0 on trace end. */ // gh-142543: inlining this function causes stack overflows @@ -999,8 +964,14 @@ _PyJit_translate_single_bytecode_to_trace( else { int extended_arg = orig_oparg > 255; uint32_t jump_target = next_inst + orig_oparg + extended_arg; - assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); + /* Jump must be to an "END" either END_FOR or END_SEND */ + assert(( + _Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR && + _Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER + ) + || + _Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_SEND + ); if (is_for_iter_test[uop]) { target = jump_target + 1; } @@ -1448,7 +1419,7 @@ allocate_executor(int exit_count, int length) res->trace = (_PyUOpInstruction *)(res->exits + exit_count); res->code_size = length; res->exit_count = exit_count; - res->jit_gdb_handle = NULL; + res->jit_registration = NULL; return res; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 1dc3a248f45f0c8..e726dc0e6fd1114 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -18,6 +18,7 @@ #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_uop_metadata.h" #include "pycore_long.h" @@ -127,7 +128,7 @@ static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, (1 << DICT_MAX_WATCHERS)); + FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, 1ULL << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, @@ -156,6 +157,17 @@ type_watcher_callback(PyTypeObject* type) return 0; } +static int +_setup_optimizer_watchers(void *Py_UNUSED(arg)) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_ATOMIC_STORE_PTR_RELEASE( + interp->dict_state.watchers[GLOBALS_WATCHER_ID], + globals_watcher_callback); + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + return 0; +} + static void watch_type(PyTypeObject *type, _PyBloomFilter *filter) { @@ -580,10 +592,8 @@ optimize_uops( // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { - interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; - interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; - } + _PyOnceFlag_CallOnce(&interp->dict_state.watcher_setup_once, + _setup_optimizer_watchers, NULL); _Py_uop_abstractcontext_init(ctx, dependencies); _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, (PyCodeObject *)func->func_code, NULL, 0); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 2dcfbb6d3418c07..edb4c644bccbf6f 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2,6 +2,8 @@ #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" +#include "pycore_stackref.h" +#include "pycore_typeobject.h" #include "pycore_uops.h" #include "pycore_uop_ids.h" #include "internal/pycore_moduleobject.h" @@ -869,15 +871,11 @@ dummy_func(void) { op(_LOAD_COMMON_CONSTANT, (-- value)) { assert(oparg < NUM_COMMON_CONSTANTS); - PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - if (_Py_IsImmortal(val)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - else { - ADD_OP(_LOAD_CONST_INLINE, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } + PyObject *val = PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg]); + assert(_Py_IsImmortal(val)); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); } op(_LOAD_SMALL_INT, (-- value)) { @@ -1452,6 +1450,29 @@ dummy_func(void) { } } + op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { + bool definite = true; + PyTypeObject *type = sym_get_type(iter); + if (type == NULL) { + type = sym_get_probable_type(iter); + definite = false; + } + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + if (!definite) { + sym_set_type(iter, type); + assert((this_instr - 1)->opcode == _RECORD_NOS_TYPE); + int32_t orig_target = (this_instr - 1)->target; + ADD_OP(_GUARD_TYPE_ITER, 0, (uintptr_t)type); + uop_buffer_last(&ctx->out_buffer)->target = orig_target; + } + ADD_OP(_ITER_NEXT_INLINE, 0, (uintptr_t)type->tp_iternext); + } + next = sym_new_not_null(ctx); + } + op(_GUARD_ITERATOR, (iterable -- iterable)) { bool definite = true; PyTypeObject *tp = sym_get_type(iterable); @@ -2019,7 +2040,16 @@ dummy_func(void) { PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *descr = _PyType_Lookup(type, name); if (descr != NULL && (Py_TYPE(descr)->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR)) { - ADD_OP(_GUARD_TYPE_VERSION, 0, type->tp_version_tag); + /* LOAD_SPECIAL expands to _RECORD_TOS_TYPE + _INSERT_NULL + + * _LOAD_SPECIAL. Insert _GUARD_TYPE_VERSION before the + * already-emitted _INSERT_NULL so deopt sees the original + * stack shape.*/ + _PyUOpInstruction *insert_null = uop_buffer_last(&ctx->out_buffer); + assert(insert_null->opcode == _INSERT_NULL); + assert(insert_null->target == this_instr->target); + REPLACE_OP(insert_null, _GUARD_TYPE_VERSION, 0, type->tp_version_tag); + ADD_OP(_INSERT_NULL, 0, 0); + bool immortal = _Py_IsImmortal(descr) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uintptr_t)descr); @@ -2345,7 +2375,7 @@ dummy_func(void) { res = sym_new_type(ctx, &PyLong_Type); Py_ssize_t length = sym_tuple_length(arg); - // Not a tuple, check if it's a const string + // Not a tuple, check if it's another immutable const with known length if (length < 0 && sym_is_const(ctx, arg)) { PyObject *const_val = sym_get_const(ctx, arg); if (const_val != NULL) { @@ -2355,6 +2385,12 @@ dummy_func(void) { else if (PyBytes_CheckExact(const_val)) { length = PyBytes_GET_SIZE(const_val); } + else if (PyFrozenDict_CheckExact(const_val)) { + length = PyDict_GET_SIZE(const_val); + } + else if (PyFrozenSet_CheckExact(const_val)) { + length = PySet_GET_SIZE(const_val); + } } } @@ -2679,7 +2715,32 @@ dummy_func(void) { stack_pointer = sym_set_stack_depth((int)this_instr->operand1, stack_pointer); } + op(_GUARD_TOS_NOT_NULL, (null_or_index -- null_or_index)) { + if (sym_is_not_null(null_or_index)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + sym_set_non_null(null_or_index); + } + } + + op(_GUARD_3OS_ASYNC_GEN_ASEND, (iter, null_or_index, value -- iter, null_or_index, value)) { + if (sym_matches_type(iter, &_PyAsyncGenASend_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + sym_set_type(iter, &_PyAsyncGenASend_Type); + } + } + op(_GET_ANEXT, (aiter -- aiter, awaitable)) { + if (sym_matches_type(aiter, &PyAsyncGen_Type)) { + awaitable = sym_new_type(ctx, &_PyAsyncGenASend_Type); + } + else { + awaitable = sym_new_not_null(ctx); + } + } // END BYTECODES // diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0942089760f6397..102b95279514141 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1860,8 +1860,15 @@ } case _GET_ANEXT: { + JitOptRef aiter; JitOptRef awaitable; - awaitable = sym_new_not_null(ctx); + aiter = stack_pointer[-1]; + if (sym_matches_type(aiter, &PyAsyncGen_Type)) { + awaitable = sym_new_type(ctx, &_PyAsyncGenASend_Type); + } + else { + awaitable = sym_new_not_null(ctx); + } CHECK_STACK_BOUNDS(1); stack_pointer[0] = awaitable; stack_pointer += 1; @@ -1876,8 +1883,6 @@ break; } - /* _SEND is not a viable micro-op for tier 2 */ - case _SEND_GEN_FRAME: { JitOptRef v; JitOptRef receiver; @@ -1896,6 +1901,58 @@ break; } + case _GUARD_TOS_IS_NONE: { + break; + } + + case _GUARD_NOS_NOT_NULL: { + JitOptRef nos; + nos = stack_pointer[-2]; + if (sym_is_not_null(nos)) { + ADD_OP(_NOP, 0, 0); + } + else { + sym_set_non_null(nos); + } + break; + } + + /* _SEND_VIRTUAL is not a viable micro-op for tier 2 */ + + case _SEND_VIRTUAL_TIER_TWO: { + JitOptRef next; + next = sym_new_not_null(ctx); + stack_pointer[-1] = next; + break; + } + + case _GUARD_3OS_ASYNC_GEN_ASEND: { + JitOptRef iter; + iter = stack_pointer[-3]; + if (sym_matches_type(iter, &_PyAsyncGenASend_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + sym_set_type(iter, &_PyAsyncGenASend_Type); + } + break; + } + + /* _SEND_ASYNC_GEN is not a viable micro-op for tier 2 */ + + case _SEND_ASYNC_GEN_TIER_TWO: { + JitOptRef asend; + JitOptRef null_out; + JitOptRef retval; + asend = sym_new_not_null(ctx); + null_out = sym_new_not_null(ctx); + retval = sym_new_not_null(ctx); + stack_pointer[-3] = asend; + stack_pointer[-2] = null_out; + stack_pointer[-1] = retval; + break; + } + case _YIELD_VALUE: { JitOptRef retval; JitOptRef value; @@ -1934,15 +1991,11 @@ case _LOAD_COMMON_CONSTANT: { JitOptRef value; assert(oparg < NUM_COMMON_CONSTANTS); - PyObject *val = _PyInterpreterState_GET()->common_consts[oparg]; - if (_Py_IsImmortal(val)) { - ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); - value = PyJitRef_Borrow(sym_new_const(ctx, val)); - } - else { - ADD_OP(_LOAD_CONST_INLINE, 0, (uintptr_t)val); - value = sym_new_const(ctx, val); - } + PyObject *val = PyStackRef_AsPyObjectBorrow( + _PyInterpreterState_GET()->common_consts[oparg]); + assert(_Py_IsImmortal(val)); + ADD_OP(_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)val); + value = PyJitRef_Borrow(sym_new_const(ctx, val)); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; @@ -3640,6 +3693,41 @@ /* _FOR_ITER is not a viable micro-op for tier 2 */ case _FOR_ITER_TIER_TWO: { + JitOptRef iter; + JitOptRef next; + iter = stack_pointer[-2]; + bool definite = true; + PyTypeObject *type = sym_get_type(iter); + if (type == NULL) { + type = sym_get_probable_type(iter); + definite = false; + } + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + if (!definite) { + sym_set_type(iter, type); + assert((this_instr - 1)->opcode == _RECORD_NOS_TYPE); + int32_t orig_target = (this_instr - 1)->target; + ADD_OP(_GUARD_TYPE_ITER, 0, (uintptr_t)type); + uop_buffer_last(&ctx->out_buffer)->target = orig_target; + } + ADD_OP(_ITER_NEXT_INLINE, 0, (uintptr_t)type->tp_iternext); + } + next = sym_new_not_null(ctx); + CHECK_STACK_BOUNDS(1); + stack_pointer[0] = next; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + break; + } + + case _GUARD_TYPE_ITER: { + break; + } + + case _ITER_NEXT_INLINE: { JitOptRef next; next = sym_new_not_null(ctx); CHECK_STACK_BOUNDS(1); @@ -3653,6 +3741,18 @@ break; } + case _GUARD_TOS_NOT_NULL: { + JitOptRef null_or_index; + null_or_index = stack_pointer[-1]; + if (sym_is_not_null(null_or_index)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + else { + sym_set_non_null(null_or_index); + } + break; + } + /* _FOR_ITER_VIRTUAL is not a viable micro-op for tier 2 */ case _FOR_ITER_VIRTUAL_TIER_TWO: { @@ -3792,7 +3892,11 @@ PyObject *name = _Py_SpecialMethods[oparg].name; PyObject *descr = _PyType_Lookup(type, name); if (descr != NULL && (Py_TYPE(descr)->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR)) { - ADD_OP(_GUARD_TYPE_VERSION, 0, type->tp_version_tag); + _PyUOpInstruction *insert_null = uop_buffer_last(&ctx->out_buffer); + assert(insert_null->opcode == _INSERT_NULL); + assert(insert_null->target == this_instr->target); + REPLACE_OP(insert_null, _GUARD_TYPE_VERSION, 0, type->tp_version_tag); + ADD_OP(_INSERT_NULL, 0, 0); bool immortal = _Py_IsImmortal(descr) || (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE); ADD_OP(immortal ? _LOAD_CONST_INLINE_BORROW : _LOAD_CONST_INLINE, 0, (uintptr_t)descr); @@ -3837,10 +3941,6 @@ break; } - case _GUARD_KEYS_VERSION: { - break; - } - case _LOAD_ATTR_METHOD_WITH_VALUES: { JitOptRef owner; JitOptRef attr; @@ -4177,18 +4277,6 @@ break; } - case _GUARD_NOS_NOT_NULL: { - JitOptRef nos; - nos = stack_pointer[-2]; - if (sym_is_not_null(nos)) { - ADD_OP(_NOP, 0, 0); - } - else { - sym_set_non_null(nos); - } - break; - } - case _GUARD_THIRD_NULL: { JitOptRef null; null = stack_pointer[-3]; @@ -4540,12 +4628,13 @@ length = PyUnicode_GET_LENGTH(const_val); } else if (PyBytes_CheckExact(const_val)) { - CHECK_STACK_BOUNDS(-2); - stack_pointer[-3] = res; - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); length = PyBytes_GET_SIZE(const_val); - stack_pointer += 2; + } + else if (PyFrozenDict_CheckExact(const_val)) { + length = PyDict_GET_SIZE(const_val); + } + else if (PyFrozenSet_CheckExact(const_val)) { + length = PySet_GET_SIZE(const_val); } } } diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 0c460282feceef9..32b147199544cfc 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -82,6 +82,9 @@ #if defined(__linux__) # include <sys/syscall.h> // System call interface #endif +#if defined(__APPLE__) +# include <mach/mach_time.h> // mach_absolute_time, mach_timebase_info +#endif // ============================================================================= // CONSTANTS AND CONFIGURATION @@ -217,11 +220,7 @@ struct BaseEvent { typedef struct { struct BaseEvent base; // Common event header uint32_t process_id; // Process ID where code was generated -#if defined(__APPLE__) - uint64_t thread_id; // Thread ID where code was generated -#else uint32_t thread_id; // Thread ID where code was generated -#endif uint64_t vma; // Virtual memory address where code is loaded uint64_t code_address; // Address of the actual machine code uint64_t code_size; // Size of the machine code in bytes @@ -295,7 +294,9 @@ static PerfMapJitState perf_jit_map_state; // ============================================================================= /* Time conversion constant */ +#if !defined(__APPLE__) static const intptr_t nanoseconds_per_second = 1000000000; +#endif /* * Get current monotonic time in nanoseconds @@ -307,6 +308,18 @@ static const intptr_t nanoseconds_per_second = 1000000000; * Returns: Current monotonic time in nanoseconds since an arbitrary epoch */ static int64_t get_current_monotonic_ticks(void) { +#if defined(__APPLE__) + // On macOS the jitdump file is consumed by profilers (such as samply) that + // timestamp their samples using mach_absolute_time(). The jitdump event + // timestamps must use the same clock domain, otherwise the JIT code + // mappings cannot be lined up with the samples. + static mach_timebase_info_data_t timebase = {0, 0}; + if (timebase.denom == 0) { + (void)mach_timebase_info(&timebase); + } + uint64_t ticks = mach_absolute_time(); + return (int64_t)(ticks * timebase.numer / timebase.denom); +#else struct timespec ts; if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { Py_UNREACHABLE(); // Should never fail on supported systems @@ -318,6 +331,7 @@ static int64_t get_current_monotonic_ticks(void) { result *= nanoseconds_per_second; result += ts.tv_nsec; return result; +#endif } /* @@ -652,7 +666,12 @@ static void perf_map_jit_write_entry_with_name( ev.base.time_stamp = get_current_monotonic_ticks(); ev.process_id = getpid(); #if defined(__APPLE__) - pthread_threadid_np(NULL, &ev.thread_id); + // The jitdump format defines the thread id field as a 32-bit value, but + // pthread_threadid_np() returns a 64-bit id. Truncate it to 32 bits to + // keep the record layout identical to other platforms. + uint64_t thread_id = 0; + pthread_threadid_np(NULL, &thread_id); + ev.thread_id = (uint32_t)thread_id; #else ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call #endif diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 58c61e64bfc4e99..d90b789c2b57126 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -210,9 +210,8 @@ enum perf_trampoline_type { static void free_code_arenas(void); static void -perf_trampoline_reset_state(void) +perf_trampoline_clear_code_watcher(void) { - free_code_arenas(); if (code_watcher_id >= 0) { PyCode_ClearWatcher(code_watcher_id); code_watcher_id = -1; @@ -220,6 +219,13 @@ perf_trampoline_reset_state(void) extra_code_index = -1; } +static void +perf_trampoline_reset_state(void) +{ + free_code_arenas(); + perf_trampoline_clear_code_watcher(); +} + static int perf_trampoline_code_watcher(PyCodeEvent event, PyCodeObject *co) { @@ -621,9 +627,10 @@ _PyPerfTrampoline_AfterFork_Child(void) // After fork, Fini may leave the old code watcher registered // if trampolined code objects from the parent still exist // (trampoline_refcount > 0). Clear it unconditionally before - // Init registers a new one, to prevent two watchers sharing - // the same globals and double-decrementing trampoline_refcount. - perf_trampoline_reset_state(); + // Init registers a new one, but keep the old arenas mapped: the + // child may still need to return through trampoline frames that + // were on the C stack at fork(). + perf_trampoline_clear_code_watcher(); _PyPerfTrampoline_Init(1); } } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 728c0acdd4df672..01ca459b2eb2b8f 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -19,6 +19,7 @@ #include "pycore_object.h" // _PyDebug_PrintTotalRefs() #include "pycore_obmalloc.h" // _PyMem_init_obmalloc() #include "pycore_optimizer.h" // _Py_Executors_InvalidateAll +#include "pycore_parking_lot.h" // _PyParkingLot #include "pycore_pathconfig.h" // _PyPathConfig_UpdateGlobal() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pylifecycle.h" // _PyErr_Print() @@ -27,9 +28,10 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_runtime_init.h" // _PyRuntimeState_INIT #include "pycore_setobject.h" // _PySet_NextEntry() +#include "pycore_stackref.h" // PyStackRef_FromPyObjectBorrow() #include "pycore_stats.h" // _PyStats_InterpInit() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() -#include "pycore_traceback.h" // _Py_DumpTracebackThreads() +#include "pycore_traceback.h" // PyUnstable_TracebackThreads() #include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_typevarobject.h" // _Py_clear_generic_types() @@ -427,7 +429,8 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config) } } - tstate->interp->long_state.max_str_digits = config->int_max_str_digits; + _Py_atomic_store_int(&tstate->interp->long_state.max_str_digits, + config->int_max_str_digits); // Update the sys module for the new configuration if (_PySys_UpdateConfig(tstate) < 0) { @@ -877,22 +880,28 @@ pycore_init_builtins(PyThreadState *tstate) goto error; } - interp->common_consts[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError; - interp->common_consts[CONSTANT_NOTIMPLEMENTEDERROR] = PyExc_NotImplementedError; - interp->common_consts[CONSTANT_BUILTIN_TUPLE] = (PyObject *)&PyTuple_Type; - interp->common_consts[CONSTANT_BUILTIN_ALL] = all; - interp->common_consts[CONSTANT_BUILTIN_ANY] = any; - interp->common_consts[CONSTANT_BUILTIN_LIST] = (PyObject *)&PyList_Type; - interp->common_consts[CONSTANT_BUILTIN_SET] = (PyObject *)&PySet_Type; - interp->common_consts[CONSTANT_NONE] = Py_None; - interp->common_consts[CONSTANT_EMPTY_STR] = + PyObject *common_objs[NUM_COMMON_CONSTANTS] = {NULL}; + common_objs[CONSTANT_ASSERTIONERROR] = PyExc_AssertionError; + common_objs[CONSTANT_NOTIMPLEMENTEDERROR] = PyExc_NotImplementedError; + common_objs[CONSTANT_BUILTIN_TUPLE] = (PyObject *)&PyTuple_Type; + common_objs[CONSTANT_BUILTIN_ALL] = all; + common_objs[CONSTANT_BUILTIN_ANY] = any; + common_objs[CONSTANT_BUILTIN_LIST] = (PyObject *)&PyList_Type; + common_objs[CONSTANT_BUILTIN_SET] = (PyObject *)&PySet_Type; + common_objs[CONSTANT_NONE] = Py_None; + common_objs[CONSTANT_EMPTY_STR] = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_STR); - interp->common_consts[CONSTANT_TRUE] = Py_True; - interp->common_consts[CONSTANT_FALSE] = Py_False; - interp->common_consts[CONSTANT_MINUS_ONE] = + common_objs[CONSTANT_TRUE] = Py_True; + common_objs[CONSTANT_FALSE] = Py_False; + common_objs[CONSTANT_MINUS_ONE] = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS - 1]; + common_objs[CONSTANT_BUILTIN_FROZENSET] = (PyObject *)&PyFrozenSet_Type; + common_objs[CONSTANT_EMPTY_TUPLE] = + Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_TUPLE); for (int i = 0; i < NUM_COMMON_CONSTANTS; i++) { - assert(interp->common_consts[i] != NULL); + assert(common_objs[i] != NULL); + _Py_SetImmortal(common_objs[i]); + interp->common_consts[i] = PyStackRef_FromPyObjectBorrow(common_objs[i]); } PyObject *list_append = _PyType_Lookup(&PyList_Type, &_Py_ID(append)); @@ -1485,15 +1494,11 @@ init_interp_main(PyThreadState *tstate) // Initialize lazy imports based on configuration. Do this after site // module is imported to avoid circular imports during startup. - if (config->lazy_imports != -1) { - PyImport_LazyImportsMode lazy_mode; - if (config->lazy_imports == 1) { - lazy_mode = PyImport_LAZY_ALL; - } - else { - lazy_mode = PyImport_LAZY_NONE; - } - if (PyImport_SetLazyImportsMode(lazy_mode) < 0) { + if (config->lazy_imports == 0) { + return _PyStatus_ERR("PyConfig.lazy_imports=0 is not supported"); + } + if (config->lazy_imports == 1) { + if (PyImport_SetLazyImportsMode(PyImport_LAZY_ALL) < 0) { return _PyStatus_ERR("failed to set lazy imports mode"); } } @@ -2229,15 +2234,13 @@ interp_has_threads(PyInterpreterState *interp) /* This needs to check for non-daemon threads only, otherwise we get stuck * in an infinite loop. */ assert(interp != NULL); - ASSERT_WORLD_STOPPED(interp); + ASSERT_HEAD_IS_LOCKED(interp->runtime); assert(interp->threads.head != NULL); if (interp->threads.head->next == NULL) { // No other threads active, easy way out. return 0; } - // We don't have to worry about locking this because the - // world is stopped. _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { if (tstate->_whence == _PyThreadState_WHENCE_THREADING) { return 1; @@ -2269,9 +2272,7 @@ static int runtime_has_subinterpreters(_PyRuntimeState *runtime) { assert(runtime != NULL); - HEAD_LOCK(runtime); PyInterpreterState *interp = runtime->interpreters.head; - HEAD_UNLOCK(runtime); return interp->next != NULL; } @@ -2280,6 +2281,7 @@ make_pre_finalization_calls(PyThreadState *tstate, int subinterpreters) { assert(tstate != NULL); PyInterpreterState *interp = tstate->interp; + assert(_Py_atomic_load_uintptr(&interp->finalization_guards) != _PyInterpreterGuard_GUARDS_NOT_ALLOWED); /* Each of these functions can start one another, e.g. a pending call * could start a thread or vice versa. To ensure that we properly clean * call everything, we run these in a loop until none of them run anything. */ @@ -2306,41 +2308,78 @@ make_pre_finalization_calls(PyThreadState *tstate, int subinterpreters) if (subinterpreters) { /* Clean up any lingering subinterpreters. - - Two preconditions need to be met here: - - - This has to happen before _PyRuntimeState_SetFinalizing is - called, or else threads might get prematurely blocked. - - The world must not be stopped, as finalizers can run. - */ + * Two preconditions need to be met here: + * 1. This has to happen before _PyRuntimeState_SetFinalizing is + * called, or else threads might get prematurely blocked. + * 2. The world must not be stopped, as finalizers can run. + */ finalize_subinterpreters(); } + // This is used as a throttle to prevent constant spinning while + // on finalization guards. + for (;;) { + uintptr_t num_guards = _Py_atomic_load_uintptr(&interp->finalization_guards); + if (num_guards == 0) { + break; + } + + int ret = _PyParkingLot_Park(&interp->finalization_guards, + &num_guards, sizeof(num_guards), -1, + NULL, /*detach=*/1); + if (ret == Py_PARK_OK) { + break; + } + else if (ret == Py_PARK_INTR) { + if (PyErr_CheckSignals() < 0) { + int fatal = PyErr_ExceptionMatches(PyExc_KeyboardInterrupt); + PyErr_FormatUnraisable("Exception ignored while waiting on finalization guards"); + if (fatal) { + fputs("Interrupted while waiting on finalization guards\n", stderr); + exit(1); + } + } + assert(!PyErr_Occurred()); + } + else { + assert(ret == Py_PARK_AGAIN); + } + } /* Stop the world to prevent other threads from creating threads or * atexit callbacks. On the default build, this is simply locked by * the GIL. For pending calls, we acquire the dedicated mutex, because * Py_AddPendingCall() can be called without an attached thread state. */ - PyMutex_Lock(&interp->ceval.pending.mutex); - // XXX Why does _PyThreadState_DeleteList() rely on all interpreters - // being stopped? _PyEval_StopTheWorldAll(interp->runtime); + + HEAD_LOCK(interp->runtime); int has_subinterpreters = subinterpreters ? runtime_has_subinterpreters(interp->runtime) : 0; + uintptr_t guards_expected = 0; int should_continue = (interp_has_threads(interp) || interp_has_atexit_callbacks(interp) || interp_has_pending_calls(interp) || has_subinterpreters); + if (!should_continue) { - break; + // We only want to prevent new guards once we're sure that we + // won't be running another pre-finalization cycle. + if (_Py_atomic_compare_exchange_uintptr(&interp->finalization_guards, + &guards_expected, + _PyInterpreterGuard_GUARDS_NOT_ALLOWED) == 1) { + HEAD_UNLOCK(interp->runtime); + break; + } } + HEAD_UNLOCK(interp->runtime); _PyEval_StartTheWorldAll(interp->runtime); PyMutex_Unlock(&interp->ceval.pending.mutex); } assert(PyMutex_IsLocked(&interp->ceval.pending.mutex)); + assert(_Py_atomic_load_uintptr(&interp->finalization_guards) == _PyInterpreterGuard_GUARDS_NOT_ALLOWED); ASSERT_WORLD_STOPPED(interp); } @@ -3289,7 +3328,9 @@ apple_log_write_impl(PyObject *self, PyObject *args) // Pass the user-provided text through explicit %s formatting // to avoid % literals being interpreted as a formatting directive. - os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + // Using {public} ensures "dynamic" string messages are visible + // in the log without special configuration. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%{public}s", text); Py_RETURN_NONE; } @@ -3348,9 +3389,9 @@ _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, /* display the current Python stack */ #ifndef Py_GIL_DISABLED - _Py_DumpTracebackThreads(fd, interp, tstate, 0); + PyUnstable_DumpTracebackThreads(fd, interp, tstate, 0); #else - _Py_DumpTraceback(fd, tstate); + PyUnstable_DumpTraceback(fd, tstate); #endif } @@ -3691,7 +3732,9 @@ fatal_error(int fd, int header, const char *prefix, const char *msg, This function already did its best to display a traceback. Disable faulthandler to prevent writing a second traceback on abort(). */ - _PyFaulthandler_Fini(); + if (has_tstate_and_gil) { + _PyFaulthandler_Fini(); + } /* Check if the current Python thread hold the GIL */ if (has_tstate_and_gil) { diff --git a/Python/pystate.c b/Python/pystate.c index 2df24597e65785a..fed1df0173bacf1 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -12,8 +12,9 @@ #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interpframe.h" // _PyThreadState_HasStackSpace() -#include "pycore_object.h" // _PyType_InitCache() +#include "pycore_object.h" // _PyType_InitCache(), _Py_ClearImmortal() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() +#include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS #include "pycore_optimizer.h" // JIT_CLEANUP_THRESHOLD #include "pycore_parking_lot.h" // _PyParkingLot_AfterFork() #include "pycore_pyerrors.h" // _PyErr_Clear() @@ -21,7 +22,7 @@ #include "pycore_pymem.h" // _PyMem_DebugEnabled() #include "pycore_runtime.h" // _PyRuntime #include "pycore_runtime_init.h" // _PyRuntimeState_INIT -#include "pycore_stackref.h" // Py_STACKREF_DEBUG +#include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() #include "pycore_stats.h" // FT_STAT_WORLD_STOP_INC() #include "pycore_time.h" // _PyTime_Init() #include "pycore_uniqueid.h" // _PyObject_FinalizePerThreadRefcounts() @@ -320,6 +321,7 @@ _Py_COMP_DIAG_POP &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ &(runtime)->_main_interpreter.code_state.mutex, \ + &(runtime)->_main_interpreter.dict_state.watcher_mutex, \ } static void @@ -633,7 +635,7 @@ init_interpreter(PyInterpreterState *interp, // Trace fitness configuration init_policy(&interp->opt_config.fitness_initial, "PYTHON_JIT_FITNESS_INITIAL", - FITNESS_INITIAL, EXIT_QUALITY_CLOSE_LOOP, UOP_MAX_TRACE_LENGTH - 1); + FITNESS_INITIAL, EXIT_QUALITY_CLOSE_LOOP, FITNESS_INITIAL); interp->opt_config.specialization_enabled = !is_env_enabled("PYTHON_SPECIALIZATION_OFF"); interp->opt_config.uops_optimize_enabled = !is_env_disabled("PYTHON_UOPS_OPTIMIZE"); @@ -777,6 +779,36 @@ extern void _Py_stackref_report_leaks(PyInterpreterState *interp); #endif +static int +common_const_is_initialized(_PyStackRef ref) +{ +#if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) + return !PyStackRef_IsNull(ref); +#else + return ref.bits != 0 && !PyStackRef_IsNull(ref); +#endif +} + + +static void +common_constants_clear(PyInterpreterState *interp) +{ + for (int i = 0; i < NUM_COMMON_CONSTANTS; i++) { + _PyStackRef ref = interp->common_consts[i]; + if (!common_const_is_initialized(ref)) { + continue; + } + PyObject *obj = PyStackRef_AsPyObjectBorrow(ref); + PyStackRef_XCLOSE(ref); + interp->common_consts[i] = PyStackRef_NULL; + // Refcount reclamation skips heap immortals; release manually. + if (_Py_IsImmortal(obj) && !_Py_IsStaticImmortal(obj)) { + _Py_ClearImmortal(obj); + } + } +} + + static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) { @@ -903,6 +935,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) PyDict_Clear(interp->builtins); Py_CLEAR(interp->sysdict); Py_CLEAR(interp->builtins); + common_constants_clear(interp); #if !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) # ifdef Py_STACKREF_CLOSE_DEBUG @@ -2889,34 +2922,40 @@ PyGILState_Check(void) return (tstate == tcur); } +static PyInterpreterGuard * +get_main_interp_guard(void) +{ + PyInterpreterView *view = PyInterpreterView_FromMain(); + if (view == NULL) { + return NULL; + } + + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + PyInterpreterView_Close(view); + return guard; +} + PyGILState_STATE PyGILState_Ensure(void) { - _PyRuntimeState *runtime = &_PyRuntime; - /* Note that we do not auto-init Python here - apart from potential races with 2 threads auto-initializing, pep-311 spells out other issues. Embedders are expected to have called Py_Initialize(). */ - /* Ensure that _PyEval_InitThreads() and _PyGILState_Init() have been - called by Py_Initialize() - - TODO: This isn't thread-safe. There's no protection here against - concurrent finalization of the interpreter; it's simply a guard - for *after* the interpreter has finalized. - */ - if (!_PyEval_ThreadsInitialized() || runtime->gilstate.autoInterpreterState == NULL) { - PyThread_hang_thread(); - } - PyThreadState *tcur = gilstate_get(); int has_gil; if (tcur == NULL) { /* Create a new Python thread state for this thread */ - // XXX Use PyInterpreterState_EnsureThreadState()? - tcur = new_threadstate(runtime->gilstate.autoInterpreterState, - _PyThreadState_WHENCE_GILSTATE); + PyInterpreterGuard *guard = get_main_interp_guard(); + if (guard == NULL) { + // The main interpreter has finished, so we don't have + // any intepreter to make a thread state for. Hang the + // thread to act as failure. + PyThread_hang_thread(); + } + tcur = new_threadstate(guard->interp, + _PyThreadState_WHENCE_C_API); if (tcur == NULL) { Py_FatalError("Couldn't create thread-state for new thread"); } @@ -2928,6 +2967,7 @@ PyGILState_Ensure(void) assert(tcur->gilstate_counter == 1); tcur->gilstate_counter = 0; has_gil = 0; /* new thread state is never current */ + PyInterpreterGuard_Close(guard); } else { has_gil = holds_gil(tcur); @@ -3309,3 +3349,277 @@ _Py_GetMainConfig(void) } return _PyInterpreterState_GetConfig(interp); } + +Py_ssize_t +_PyInterpreterState_GuardCountdown(PyInterpreterState *interp) +{ + assert(interp != NULL); + Py_ssize_t count = _Py_atomic_load_uintptr(&interp->finalization_guards); + assert(count >= 0); + return count; +} + +PyInterpreterState * +_PyInterpreterGuard_GetInterpreter(PyInterpreterGuard *guard) +{ + assert(guard != NULL); + assert(guard->interp != NULL); + return guard->interp; +} + +static int +try_acquire_interp_guard(PyInterpreterState *interp, PyInterpreterGuard *guard) +{ + assert(interp != NULL); + + uintptr_t expected; + do { + expected = _Py_atomic_load_uintptr(&interp->finalization_guards); + if (expected == _PyInterpreterGuard_GUARDS_NOT_ALLOWED) { + return -1; + } + } while (_Py_atomic_compare_exchange_uintptr(&interp->finalization_guards, + &expected, + expected + 1) == 0); + assert(_Py_atomic_load_uintptr(&interp->finalization_guards) > 0); + assert(_Py_atomic_load_uintptr(&interp->finalization_guards) != _PyInterpreterGuard_GUARDS_NOT_ALLOWED); + + guard->interp = interp; + return 0; +} + +PyInterpreterGuard * +PyInterpreterGuard_FromCurrent(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp != NULL); + + PyInterpreterGuard *guard = PyMem_RawMalloc(sizeof(PyInterpreterGuard)); + if (guard == NULL) { + PyErr_NoMemory(); + return NULL; + } + + if (try_acquire_interp_guard(interp, guard) < 0) { + PyMem_RawFree(guard); + PyErr_SetString(PyExc_PythonFinalizationError, + "cannot acquire finalization guard anymore"); + return NULL; + } + + return guard; +} + +void +PyInterpreterGuard_Close(PyInterpreterGuard *guard) +{ + PyInterpreterState *interp = guard->interp; + assert(interp != NULL); + + assert(_Py_atomic_load_uintptr(&interp->finalization_guards) != _PyInterpreterGuard_GUARDS_NOT_ALLOWED); + uintptr_t old_value = _Py_atomic_add_uintptr(&interp->finalization_guards, -1); + if (old_value == 1) { + _PyParkingLot_UnparkAll(&interp->finalization_guards); + } + + assert(old_value > 0); + PyMem_RawFree(guard); +} + +PyInterpreterView * +PyInterpreterView_FromCurrent(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(interp != NULL); + + // PyInterpreterView_Close() can be called without an attached thread + // state, so we have to use the raw allocator. + PyInterpreterView *view = PyMem_RawMalloc(sizeof(PyInterpreterView)); + if (view == NULL) { + PyErr_NoMemory(); + return NULL; + } + + view->id = interp->id; + return view; +} + +void +PyInterpreterView_Close(PyInterpreterView *view) +{ + assert(view != NULL); + PyMem_RawFree(view); +} + +PyInterpreterGuard * +PyInterpreterGuard_FromView(PyInterpreterView *view) +{ + assert(view != NULL); + int64_t interp_id = view->id; + assert(interp_id >= 0); + + // This allocation has to happen before we acquire the runtime lock, because + // PyMem_RawMalloc() might call some weird callback (such as tracemalloc) + // that tries to re-entrantly acquire the lock. + PyInterpreterGuard *guard = PyMem_RawMalloc(sizeof(PyInterpreterGuard)); + if (guard == NULL) { + return NULL; + } + + // Interpreters cannot be deleted while we hold the runtime lock. + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyInterpreterState *interp = interp_look_up_id(runtime, interp_id); + if (interp == NULL) { + HEAD_UNLOCK(runtime); + PyMem_RawFree(guard); + return NULL; + } + + int result = try_acquire_interp_guard(interp, guard); + HEAD_UNLOCK(runtime); + + if (result < 0) { + PyMem_RawFree(guard); + return NULL; + } + + assert(guard->interp != NULL); + return guard; +} + +PyInterpreterView * +PyInterpreterView_FromMain(void) +{ + PyInterpreterView *view = PyMem_RawMalloc(sizeof(PyInterpreterView)); + if (view == NULL) { + return NULL; + } + + // The main interpreter always has an ID of zero. + view->id = 0; + + return view; +} + +static const PyThreadStateToken *_no_tstate_sentinel = (const PyThreadStateToken *)&_no_tstate_sentinel; +#define NO_TSTATE_SENTINEL ((PyThreadStateToken *)_no_tstate_sentinel) + +PyThreadStateToken * +PyThreadState_Ensure(PyInterpreterGuard *guard) +{ + assert(guard != NULL); + PyInterpreterState *interp = guard->interp; + assert(interp != NULL); + PyThreadState *attached_tstate = current_fast_get(); + if (attached_tstate != NULL && attached_tstate->interp == interp) { + /* Yay! We already have an attached thread state that matches. */ + ++attached_tstate->ensure.counter; + return attached_tstate; + } + + PyThreadState *detached_gilstate = gilstate_get(); + if (detached_gilstate != NULL && detached_gilstate->interp == interp) { + /* There's a detached thread state that works. */ + assert(attached_tstate == NULL); + ++detached_gilstate->ensure.counter; + _PyThreadState_Attach(detached_gilstate); + return NO_TSTATE_SENTINEL; + } + + PyThreadState *fresh_tstate = _PyThreadState_NewBound(interp, + _PyThreadState_WHENCE_C_API); + if (fresh_tstate == NULL) { + return NULL; + } + fresh_tstate->ensure.counter = 1; + fresh_tstate->ensure.delete_on_release = 1; + + if (attached_tstate != NULL) { + return (PyThreadStateToken *)PyThreadState_Swap(fresh_tstate); + } + + _PyThreadState_Attach(fresh_tstate); + return NO_TSTATE_SENTINEL; +} + +PyThreadStateToken * +PyThreadState_EnsureFromView(PyInterpreterView *view) +{ + assert(view != NULL); + PyInterpreterGuard *guard = PyInterpreterGuard_FromView(view); + if (guard == NULL) { + return NULL; + } + + PyThreadStateToken *result = (PyThreadStateToken *)PyThreadState_Ensure(guard); + if (result == NULL) { + PyInterpreterGuard_Close(guard); + return NULL; + } + + PyThreadState *tstate = current_fast_get(); + assert(tstate != NULL); + + if (tstate->ensure.owned_guard != NULL) { + assert(tstate->ensure.owned_guard->interp == guard->interp); + PyInterpreterGuard_Close(guard); + } + else { + assert(tstate->ensure.owned_guard == NULL); + tstate->ensure.owned_guard = guard; + } + + return result; +} + +void +PyThreadState_Release(PyThreadStateToken *token) +{ + PyThreadState *tstate = current_fast_get(); + _Py_EnsureTstateNotNULL(tstate); + Py_ssize_t remaining = --tstate->ensure.counter; + if (remaining < 0) { + Py_FatalError("PyThreadState_Release() called more times than PyThreadState_Ensure()"); + } + + if (remaining != 0) { + // If the corresponding PyThreadState_Ensure() call used a detached + // thread state, we want to detach it again. + if (token == NO_TSTATE_SENTINEL) { + PyThreadState_Swap(NULL); + } + return; + } + + PyThreadState *to_restore; + if (token == NO_TSTATE_SENTINEL) { + to_restore = NULL; + } + else { + to_restore = (PyThreadState *)token; + } + + PyInterpreterGuard *owned_guard = tstate->ensure.owned_guard; + assert(tstate->ensure.delete_on_release == 1 || tstate->ensure.delete_on_release == 0); + if (tstate->ensure.delete_on_release) { + ++tstate->ensure.counter; + PyThreadState_Clear(tstate); + --tstate->ensure.counter; + } + else if (owned_guard != NULL) { + tstate->ensure.owned_guard = NULL; + } + + PyThreadState *check_tstate = PyThreadState_Swap(to_restore); + (void)check_tstate; + assert(check_tstate == tstate); + + if (tstate->ensure.delete_on_release) { + PyThreadState_Delete(tstate); + } + + if (owned_guard != NULL) { + PyInterpreterGuard_Close(owned_guard); + } +} diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 645bb013581288e..8fb1fa36f85e739 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -36,7 +36,7 @@ _Py_hexlify_scalar(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) adds a ton of complication. Who ever really hexes huge data? The 16-64 byte boosts align nicely with md5 - sha512 hexdigests. */ -#ifdef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR +#ifdef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR /* 128-bit vector of 16 unsigned bytes */ typedef unsigned char v16u8 __attribute__((vector_size(16))); @@ -110,7 +110,7 @@ _Py_hexlify_simd(const unsigned char *src, Py_UCS1 *dst, Py_ssize_t len) _Py_hexlify_scalar(src + i, dst, len - i); } -#endif /* HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ +#endif /* _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR */ static PyObject * _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, @@ -191,7 +191,7 @@ _Py_strhex_impl(const char* argbuf, Py_ssize_t arglen, unsigned char c; if (bytes_per_sep_group == 0) { -#ifdef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR +#ifdef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR if (arglen >= 16) { _Py_hexlify_simd((const unsigned char *)argbuf, retbuf, arglen); } diff --git a/Python/record_functions.c.h b/Python/record_functions.c.h index 13996b30b9c03c7..98abe3d0505e202 100644 --- a/Python/record_functions.c.h +++ b/Python/record_functions.c.h @@ -103,10 +103,9 @@ void _PyOpcode_RecordFunction_CODE(_PyInterpreterFrame *frame, _PyStackRef *stac #define _RECORD_NOS_TYPE_INDEX 3 #define _RECORD_3OS_GEN_FUNC_INDEX 4 #define _RECORD_TOS_INDEX 5 -#define _RECORD_NOS_GEN_FUNC_INDEX 6 -#define _RECORD_CALLABLE_INDEX 7 -#define _RECORD_CALLABLE_KW_INDEX 8 -#define _RECORD_4OS_INDEX 9 +#define _RECORD_CALLABLE_INDEX 6 +#define _RECORD_CALLABLE_KW_INDEX 7 +#define _RECORD_4OS_INDEX 8 const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [TO_BOOL_BOOL] = {1, {_RECORD_TOS_TYPE_INDEX}}, @@ -138,6 +137,9 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [STORE_SUBSCR_DICT] = {1, {_RECORD_NOS_TYPE_INDEX}}, [SEND] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, [SEND_GEN] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, + [FOR_ITER] = {1, {_RECORD_NOS_INDEX}}, + [SEND_VIRTUAL] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, + [SEND_ASYNC_GEN] = {1, {_RECORD_3OS_GEN_FUNC_INDEX}}, [STORE_ATTR] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_SUPER_ATTR] = {1, {_RECORD_NOS_INDEX}}, [LOAD_SUPER_ATTR_METHOD] = {1, {_RECORD_NOS_INDEX}}, @@ -156,12 +158,11 @@ const _PyOpcodeRecordEntry _PyOpcode_RecordEntries[256] = { [GET_ITER] = {1, {_RECORD_TOS_TYPE_INDEX}}, [GET_ITER_SELF] = {1, {_RECORD_TOS_TYPE_INDEX}}, [GET_ITER_VIRTUAL] = {1, {_RECORD_TOS_TYPE_INDEX}}, - [FOR_ITER] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, - [FOR_ITER_VIRTUAL] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, - [FOR_ITER_LIST] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, - [FOR_ITER_TUPLE] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, - [FOR_ITER_RANGE] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, - [FOR_ITER_GEN] = {1, {_RECORD_NOS_GEN_FUNC_INDEX}}, + [FOR_ITER_VIRTUAL] = {1, {_RECORD_NOS_INDEX}}, + [FOR_ITER_LIST] = {1, {_RECORD_NOS_INDEX}}, + [FOR_ITER_TUPLE] = {1, {_RECORD_NOS_INDEX}}, + [FOR_ITER_RANGE] = {1, {_RECORD_NOS_INDEX}}, + [FOR_ITER_GEN] = {1, {_RECORD_NOS_INDEX}}, [LOAD_SPECIAL] = {1, {_RECORD_TOS_TYPE_INDEX}}, [LOAD_ATTR_METHOD_WITH_VALUES] = {1, {_RECORD_TOS_INDEX}}, [LOAD_ATTR_METHOD_NO_DICT] = {1, {_RECORD_TOS_INDEX}}, @@ -205,6 +206,7 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [BINARY_OP_SUBSCR_GETITEM] = {1, 0, {0}}, [STORE_SUBSCR_DICT] = {1, 0, {0}}, [SEND_GEN] = {1, 0, {0}}, + [FOR_ITER] = {1, 1, {0}}, [LOAD_SUPER_ATTR_METHOD] = {1, 0, {0}}, [LOAD_ATTR_INSTANCE_VALUE] = {1, 1, {0}}, [LOAD_ATTR_WITH_HINT] = {1, 1, {0}}, @@ -219,7 +221,7 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [GET_ITER] = {1, 0, {0}}, [GET_ITER_SELF] = {1, 0, {0}}, [GET_ITER_VIRTUAL] = {1, 0, {0}}, - [FOR_ITER_GEN] = {1, 0, {0}}, + [FOR_ITER_GEN] = {1, 1, {0}}, [LOAD_SPECIAL] = {1, 0, {0}}, [LOAD_ATTR_METHOD_WITH_VALUES] = {1, 1, {0}}, [LOAD_ATTR_METHOD_NO_DICT] = {1, 1, {0}}, @@ -245,32 +247,89 @@ const _PyOpcodeRecordSlotMap _PyOpcode_RecordSlotMaps[256] = { [BINARY_OP] = {2, 2, {1, 0}}, }; -const _Py_RecordFuncPtr _PyOpcode_RecordFunctions[10] = { +const _Py_RecordFuncPtr _PyOpcode_RecordFunctions[9] = { [0] = NULL, [_RECORD_TOS_TYPE_INDEX] = _PyOpcode_RecordFunction_TOS_TYPE, [_RECORD_NOS_INDEX] = _PyOpcode_RecordFunction_NOS, [_RECORD_NOS_TYPE_INDEX] = _PyOpcode_RecordFunction_NOS_TYPE, [_RECORD_3OS_GEN_FUNC_INDEX] = _PyOpcode_RecordFunction_3OS_GEN_FUNC, [_RECORD_TOS_INDEX] = _PyOpcode_RecordFunction_TOS, - [_RECORD_NOS_GEN_FUNC_INDEX] = _PyOpcode_RecordFunction_NOS_GEN_FUNC, [_RECORD_CALLABLE_INDEX] = _PyOpcode_RecordFunction_CALLABLE, [_RECORD_CALLABLE_KW_INDEX] = _PyOpcode_RecordFunction_CALLABLE_KW, [_RECORD_4OS_INDEX] = _PyOpcode_RecordFunction_4OS, }; +static PyObject * +_PyOpcode_RecordTransform_NOS_TYPE(PyObject *recorded_value) +{ + PyObject *transformed_value = NULL; + _PyStackRef nos; + nos = PyStackRef_FromPyObjectBorrow(recorded_value); + transformed_value = (PyObject *)Py_TYPE(PyStackRef_AsPyObjectBorrow(nos)); + Py_XINCREF(transformed_value); + Py_DECREF(recorded_value); + return transformed_value; +} + +static PyObject * +_PyOpcode_RecordTransform_TOS_TYPE(PyObject *recorded_value) +{ + PyObject *transformed_value = NULL; + _PyStackRef tos; + tos = PyStackRef_FromPyObjectBorrow(recorded_value); + transformed_value = (PyObject *)Py_TYPE(PyStackRef_AsPyObjectBorrow(tos)); + Py_XINCREF(transformed_value); + Py_DECREF(recorded_value); + return transformed_value; +} + +static PyObject * +_PyOpcode_RecordTransform_NOS_GEN_FUNC(PyObject *recorded_value) +{ + PyObject *transformed_value = NULL; + _PyStackRef nos; + nos = PyStackRef_FromPyObjectBorrow(recorded_value); + PyObject *obj = PyStackRef_AsPyObjectBorrow(nos); + if (PyGen_Check(obj)) { + PyGenObject *gen = (PyGenObject *)obj; + _PyStackRef func = gen->gi_iframe.f_funcobj; + if (!PyStackRef_IsNull(func)) { + transformed_value = (PyObject *)PyStackRef_AsPyObjectBorrow(func); + Py_XINCREF(transformed_value); + } + } + Py_DECREF(recorded_value); + return transformed_value; +} + +static PyObject * +_PyOpcode_RecordTransform_BOUND_METHOD(PyObject *recorded_value) +{ + PyObject *transformed_value = NULL; + _PyStackRef callable; + callable = PyStackRef_FromPyObjectBorrow(recorded_value); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) == &PyMethod_Type) { + transformed_value = (PyObject *)callable_o; + Py_XINCREF(transformed_value); + } + Py_DECREF(recorded_value); + return transformed_value; +} + PyObject * _PyOpcode_RecordTransformValue(int uop, PyObject *value) { - switch (uop) { - case _RECORD_TOS_TYPE: - case _RECORD_NOS_TYPE: - return record_trace_transform_to_type(value); - case _RECORD_NOS_GEN_FUNC: - case _RECORD_3OS_GEN_FUNC: - return record_trace_transform_gen_func(value); - case _RECORD_BOUND_METHOD: - return record_trace_transform_bound_method(value); - default: - return value; - } + switch (uop) { + case _RECORD_NOS_TYPE: + return _PyOpcode_RecordTransform_NOS_TYPE(value); + case _RECORD_TOS_TYPE: + return _PyOpcode_RecordTransform_TOS_TYPE(value); + case _RECORD_NOS_GEN_FUNC: + return _PyOpcode_RecordTransform_NOS_GEN_FUNC(value); + case _RECORD_BOUND_METHOD: + return _PyOpcode_RecordTransform_BOUND_METHOD(value); + default: + return value; + } } diff --git a/Python/remote_debug.h b/Python/remote_debug.h index 7628fb04ba5bae9..6fecc23502b46ef 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -100,9 +100,16 @@ extern "C" { # define HAVE_PROCESS_VM_READV 0 #endif +static inline int +_Py_RemoteDebug_HasPermissionError(void) +{ + return PyErr_Occurred() + && PyErr_ExceptionMatches(PyExc_PermissionError); +} + #define _set_debug_exception_cause(exception, format, ...) \ do { \ - if (!PyErr_ExceptionMatches(PyExc_PermissionError)) { \ + if (!_Py_RemoteDebug_HasPermissionError()) { \ PyThreadState *tstate = _PyThreadState_GET(); \ if (!_PyErr_Occurred(tstate)) { \ _PyErr_Format(tstate, exception, format, ##__VA_ARGS__); \ @@ -112,6 +119,20 @@ extern "C" { } \ } while (0) +#define _set_debug_oserror_from_errno(err, format, ...) \ + do { \ + errno = (err); \ + PyErr_SetFromErrno(PyExc_OSError); \ + _set_debug_exception_cause(PyExc_OSError, format, ##__VA_ARGS__); \ + } while (0) + +#define _set_debug_oserror_from_errno_with_filename(err, filename, format, ...) \ + do { \ + errno = (err); \ + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); \ + _set_debug_exception_cause(PyExc_OSError, format, ##__VA_ARGS__); \ + } while (0) + static inline size_t get_page_size(void) { size_t page_size = 0; @@ -147,6 +168,7 @@ typedef struct { int memfd; #endif page_cache_entry_t pages[MAX_PAGES]; + int page_cache_count; Py_ssize_t page_size; } proc_handle_t; @@ -169,7 +191,9 @@ _Py_RemoteDebug_ValidatePyRuntimeCookie(proc_handle_t *handle, uintptr_t address } char buf[sizeof(_Py_Debug_Cookie) - 1]; if (_Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(buf), buf) != 0) { - PyErr_Clear(); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyErr_Clear(); + } return 0; } return memcmp(buf, _Py_Debug_Cookie, sizeof(buf)) == 0; @@ -185,14 +209,16 @@ _Py_RemoteDebug_FreePageCache(proc_handle_t *handle) handle->pages[i].data = NULL; handle->pages[i].valid = 0; } + handle->page_cache_count = 0; } UNUSED static void _Py_RemoteDebug_ClearCache(proc_handle_t *handle) { - for (int i = 0; i < MAX_PAGES; i++) { + for (int i = 0; i < handle->page_cache_count; i++) { handle->pages[i].valid = 0; } + handle->page_cache_count = 0; } #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX @@ -202,6 +228,21 @@ static mach_port_t pid_to_task(pid_t pid); // Initialize the process handle UNUSED static int _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { + handle->pid = 0; +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + handle->task = 0; +#elif defined(MS_WINDOWS) + handle->hProcess = NULL; +#elif defined(__linux__) + handle->memfd = -1; +#endif + handle->page_size = get_page_size(); + handle->page_cache_count = 0; + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } + handle->pid = pid; #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX handle->task = pid_to_task(handle->pid); @@ -214,18 +255,12 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { PROCESS_VM_READ | PROCESS_VM_WRITE | PROCESS_VM_OPERATION | PROCESS_QUERY_INFORMATION | PROCESS_SUSPEND_RESUME, FALSE, pid); if (handle->hProcess == NULL) { - PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize Windows process handle"); return -1; } -#elif defined(__linux__) - handle->memfd = -1; #endif - handle->page_size = get_page_size(); - for (int i = 0; i < MAX_PAGES; i++) { - handle->pages[i].data = NULL; - handle->pages[i].valid = 0; - } return 0; } @@ -390,17 +425,19 @@ return_section_address_fat( size_t cpu_size = sizeof(cpu), abi64_size = sizeof(is_abi64); if (sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0) != 0) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno(err, "Failed to determine CPU type via sysctlbyname " "for fat binary analysis at 0x%lx: %s", - base, strerror(errno)); + base, strerror(err)); return 0; } if (sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0) != 0) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno(err, "Failed to determine CPU ABI capability via sysctlbyname " "for fat binary analysis at 0x%lx: %s", - base, strerror(errno)); + base, strerror(err)); return 0; } @@ -453,26 +490,29 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ { int fd = open(path, O_RDONLY); if (fd == -1) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, "Cannot open binary file '%s' for section '%s' search: %s", - path, secname, strerror(errno)); + path, secname, strerror(err)); return 0; } struct stat fs; if (fstat(fd, &fs) == -1) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, "Cannot get file size for binary '%s' during section '%s' search: %s", - path, secname, strerror(errno)); + path, secname, strerror(err)); close(fd); return 0; } void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, "Cannot memory map binary file '%s' (size: %lld bytes) for section '%s' search: %s", - path, (long long)fs.st_size, secname, strerror(errno)); + path, (long long)fs.st_size, secname, strerror(err)); close(fd); return 0; } @@ -501,15 +541,21 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ } if (munmap(map, fs.st_size) != 0) { - PyErr_Format(PyExc_OSError, - "Failed to unmap binary file '%s' (size: %lld bytes): %s", - path, (long long)fs.st_size, strerror(errno)); + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, + "Failed to unmap binary file '%s' (size: %lld bytes): %s", + path, (long long)fs.st_size, strerror(err)); + } result = 0; } if (close(fd) != 0) { - PyErr_Format(PyExc_OSError, - "Failed to close binary file '%s': %s", - path, strerror(errno)); + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, path, + "Failed to close binary file '%s': %s", + path, strerror(err)); + } result = 0; } return result; @@ -554,14 +600,15 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s char map_filename[MAXPATHLEN + 1]; - while (mach_vm_region( - proc_ref, - &address, - &size, - VM_REGION_BASIC_INFO_64, - (vm_region_info_t)&region_info, - &count, - &object_name) == KERN_SUCCESS) + kern_return_t kr; + while ((kr = mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)&region_info, + &count, + &object_name)) == KERN_SUCCESS) { if ((region_info.protection & VM_PROT_READ) == 0 @@ -585,18 +632,32 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s } if (strncmp(filename, substr, strlen(substr)) == 0) { + PyErr_Clear(); uintptr_t result = search_section_in_file( secname, map_filename, address, size, proc_ref); - if (result != 0 - && (validator == NULL || validator(handle, result))) - { - return result; + if (result != 0) { + if (validator == NULL || validator(handle, result)) { + return result; + } + if (_Py_RemoteDebug_HasPermissionError()) { + return 0; + } + } + else if (_Py_RemoteDebug_HasPermissionError()) { + return 0; } } address += size; } + if (kr != KERN_INVALID_ADDRESS && !PyErr_Occurred()) { + PyErr_Format(PyExc_RuntimeError, + "mach_vm_region failed while searching PID %d for section '%s' " + "(kern_return_t: %d)", + handle->pid, secname, kr); + } + return 0; } @@ -619,25 +680,29 @@ search_elf_file_for_section( int fd = open(elf_file, O_RDONLY); if (fd < 0) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, "Cannot open ELF file '%s' for section '%s' search: %s", - elf_file, secname, strerror(errno)); + elf_file, secname, strerror(err)); goto exit; } struct stat file_stats; if (fstat(fd, &file_stats) != 0) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, "Cannot get file size for ELF file '%s' during section '%s' search: %s", - elf_file, secname, strerror(errno)); + elf_file, secname, strerror(err)); goto exit; } file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (file_memory == MAP_FAILED) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, "Cannot memory map ELF file '%s' (size: %lld bytes) for section '%s' search: %s", - elf_file, (long long)file_stats.st_size, secname, strerror(errno)); + elf_file, (long long)file_stats.st_size, secname, strerror(err)); + file_memory = NULL; goto exit; } @@ -694,12 +759,23 @@ search_elf_file_for_section( exit: if (file_memory != NULL) { - munmap(file_memory, file_stats.st_size); + if (munmap(file_memory, file_stats.st_size) != 0) { + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, + "Failed to unmap ELF file '%s' (size: %lld bytes): %s", + elf_file, (long long)file_stats.st_size, strerror(err)); + } + result = 0; + } } if (fd >= 0 && close(fd) != 0) { - PyErr_Format(PyExc_OSError, - "Failed to close ELF file '%s': %s", - elf_file, strerror(errno)); + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, elf_file, + "Failed to close ELF file '%s': %s", + elf_file, strerror(err)); + } result = 0; } return result; @@ -714,9 +790,10 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c FILE* maps_file = fopen(maps_file_path, "r"); if (maps_file == NULL) { - PyErr_Format(PyExc_OSError, + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, maps_file_path, "Cannot open process memory map file '%s' for PID %d section search: %s", - maps_file_path, handle->pid, strerror(errno)); + maps_file_path, handle->pid, strerror(err)); return 0; } @@ -781,21 +858,39 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c } if (strstr(filename, substr)) { + PyErr_Clear(); retval = search_elf_file_for_section(handle, secname, start, path); - if (retval - && (validator == NULL || validator(handle, retval))) - { + if (retval) { + if (validator == NULL || validator(handle, retval)) { + break; + } + if (_Py_RemoteDebug_HasPermissionError()) { + retval = 0; + break; + } + } + else if (_Py_RemoteDebug_HasPermissionError()) { break; } retval = 0; } } + if (retval == 0 && !PyErr_Occurred() && ferror(maps_file)) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, maps_file_path, + "Failed to read process map file '%s' for PID %d section search: %s", + maps_file_path, handle->pid, strerror(err)); + } + PyMem_Free(line); if (fclose(maps_file) != 0) { - PyErr_Format(PyExc_OSError, - "Failed to close process map file '%s': %s", - maps_file_path, strerror(errno)); + if (!PyErr_Occurred()) { + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, maps_file_path, + "Failed to close process map file '%s': %s", + maps_file_path, strerror(err)); + } retval = 0; } @@ -818,9 +913,9 @@ static int is_process_alive(HANDLE hProcess) { static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* secname) { HANDLE hFile = CreateFileW(mod_path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); - PyErr_Format(PyExc_OSError, + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, "Cannot open PE file for section '%s' analysis (error %lu)", secname, error); return NULL; @@ -828,9 +923,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* HANDLE hMap = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, 0); if (!hMap) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); - PyErr_Format(PyExc_OSError, + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, "Cannot create file mapping for PE file section '%s' analysis (error %lu)", secname, error); CloseHandle(hFile); @@ -839,9 +934,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* BYTE* mapView = (BYTE*)MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0); if (!mapView) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); - PyErr_Format(PyExc_OSError, + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, "Cannot map view of PE file for section '%s' analysis (error %lu)", secname, error); CloseHandle(hMap); @@ -899,9 +994,9 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const } while (hProcSnap == INVALID_HANDLE_VALUE && GetLastError() == ERROR_BAD_LENGTH); if (hProcSnap == INVALID_HANDLE_VALUE) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); - PyErr_Format(PyExc_PermissionError, + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, "Unable to create module snapshot for PID %d section '%s' " "search (error %lu). Check permissions or PID validity", handle->pid, secname, error); @@ -912,17 +1007,46 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const moduleEntry.dwSize = sizeof(moduleEntry); void* runtime_addr = NULL; - for (BOOL hasModule = Module32FirstW(hProcSnap, &moduleEntry); hasModule; hasModule = Module32NextW(hProcSnap, &moduleEntry)) { + if (!Module32FirstW(hProcSnap, &moduleEntry)) { + DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, + "Unable to enumerate modules for PID %d section '%s' " + "search (error %lu)", + handle->pid, secname, error); + CloseHandle(hProcSnap); + return 0; + } + + do { // Look for either python executable or DLL if (wcsstr(moduleEntry.szModule, substr)) { + PyErr_Clear(); void *candidate = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname); - if (candidate != NULL - && (validator == NULL || validator(handle, (uintptr_t)candidate))) - { - runtime_addr = candidate; + if (candidate != NULL) { + if (validator == NULL || validator(handle, (uintptr_t)candidate)) { + runtime_addr = candidate; + break; + } + if (_Py_RemoteDebug_HasPermissionError()) { + break; + } + } + else if (_Py_RemoteDebug_HasPermissionError()) { break; } } + } while (Module32NextW(hProcSnap, &moduleEntry)); + + if (runtime_addr == NULL && !PyErr_Occurred()) { + DWORD error = GetLastError(); + if (error != ERROR_NO_MORE_FILES) { + PyErr_SetFromWindowsErr(error); + _set_debug_exception_cause(PyExc_OSError, + "Module enumeration failed for PID %d section '%s' " + "search (error %lu)", + handle->pid, secname, error); + } } CloseHandle(hProcSnap); @@ -943,24 +1067,28 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) address = search_windows_map_for_section(handle, "PyRuntime", L"python", _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_Format(PyExc_RuntimeError, - "Failed to find the PyRuntime section in process %d on Windows platform", - handle->pid); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d on Windows platform", + handle->pid); + _PyErr_ChainExceptions1(exc); + } } #elif defined(__linux__) && HAVE_PROCESS_VM_READV // On Linux, search for 'python' in executable or DLL address = search_linux_map_for_section(handle, "PyRuntime", "python", _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { - // Error out: 'python' substring covers both executable and DLL - PyObject *exc = PyErr_GetRaisedException(); - PyErr_Format(PyExc_RuntimeError, - "Failed to find the PyRuntime section in process %d on Linux platform", - handle->pid); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d on Linux platform", + handle->pid); + _PyErr_ChainExceptions1(exc); + } } #elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python @@ -969,17 +1097,19 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) PyErr_Clear(); address = search_map_for_section(handle, "PyRuntime", *candidate, _Py_RemoteDebug_ValidatePyRuntimeCookie); - if (address != 0) { + if (address != 0 || _Py_RemoteDebug_HasPermissionError()) { break; } } if (address == 0) { - PyObject *exc = PyErr_GetRaisedException(); - PyErr_Format(PyExc_RuntimeError, - "Failed to find the PyRuntime section in process %d " - "on macOS platform (tried both libpython and python)", - handle->pid); - _PyErr_ChainExceptions1(exc); + if (!_Py_RemoteDebug_HasPermissionError()) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d " + "on macOS platform (tried both libpython and python)", + handle->pid); + _PyErr_ChainExceptions1(exc); + } } #else _set_debug_exception_cause(PyExc_RuntimeError, @@ -1000,9 +1130,9 @@ open_proc_mem_fd(proc_handle_t *handle) handle->memfd = open(mem_file_path, O_RDWR); if (handle->memfd == -1) { - PyErr_SetFromErrno(PyExc_OSError); - _set_debug_exception_cause(PyExc_OSError, - "failed to open file %s: %s", mem_file_path, strerror(errno)); + int err = errno; + _set_debug_oserror_from_errno_with_filename(err, mem_file_path, + "failed to open file %s: %s", mem_file_path, strerror(err)); return -1; } return 0; @@ -1013,6 +1143,9 @@ open_proc_mem_fd(proc_handle_t *handle) static int read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) { + if (len == 0) { + return 0; + } if (handle->memfd == -1) { if (open_proc_mem_fd(handle) < 0) { return -1; @@ -1030,14 +1163,23 @@ read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, siz read_bytes = preadv(handle->memfd, local, 1, offset); if (read_bytes < 0) { + int err = errno; + errno = err; PyErr_SetFromErrno(PyExc_OSError); _set_debug_exception_cause(PyExc_OSError, "preadv failed for PID %d at address 0x%lx " "(size %zu, partial read %zd bytes): %s", - handle->pid, remote_address + result, len - result, result, strerror(errno)); + handle->pid, remote_address + result, len - result, result, strerror(err)); return -1; } + if (read_bytes == 0) { + PyErr_Format(PyExc_OSError, + "preadv returned 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += read_bytes; } while ((size_t)read_bytes != local[0].iov_len); return 0; @@ -1049,11 +1191,15 @@ read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, siz static int _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) { + if (len == 0) { + return 0; + } #ifdef MS_WINDOWS SIZE_T read_bytes = 0; SIZE_T result = 0; do { if (!ReadProcessMemory(handle->hProcess, (LPCVOID)(remote_address + result), (char*)dst + result, len - result, &read_bytes)) { + DWORD error = GetLastError(); // Check if the process is still alive: we need to be able to tell our caller // that the process is dead and not just that the read failed. if (!is_process_alive(handle->hProcess)) { @@ -1061,14 +1207,20 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address PyErr_SetFromErrno(PyExc_OSError); return -1; } - PyErr_SetFromWindowsErr(0); - DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); _set_debug_exception_cause(PyExc_OSError, "ReadProcessMemory failed for PID %d at address 0x%lx " "(size %zu, partial read %zu bytes): Windows error %lu", handle->pid, remote_address + result, len - result, result, error); return -1; } + if (read_bytes == 0) { + PyErr_Format(PyExc_OSError, + "ReadProcessMemory returned 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial read %zu bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += read_bytes; } while (result < len); return 0; @@ -1089,31 +1241,40 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address read_bytes = process_vm_readv(handle->pid, local, 1, remote, 1, 0); if (read_bytes < 0) { - if (errno == ENOSYS) { + int err = errno; + if (err == ENOSYS) { return read_remote_memory_fallback(handle, remote_address, len, dst); } + errno = err; PyErr_SetFromErrno(PyExc_OSError); - if (errno == ESRCH) { + if (err == ESRCH) { return -1; } _set_debug_exception_cause(PyExc_OSError, "process_vm_readv failed for PID %d at address 0x%lx " "(size %zu, partial read %zd bytes): %s", - handle->pid, remote_address + result, len - result, result, strerror(errno)); + handle->pid, remote_address + result, len - result, result, strerror(err)); return -1; } + if (read_bytes == 0) { + PyErr_Format(PyExc_OSError, + "process_vm_readv returned 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += read_bytes; } while ((size_t)read_bytes != local[0].iov_len); return 0; #elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX - Py_ssize_t result = -1; + mach_vm_size_t bytes_read = 0; kern_return_t kr = mach_vm_read_overwrite( handle->task, (mach_vm_address_t)remote_address, len, (mach_vm_address_t)dst, - (mach_vm_size_t*)&result); + &bytes_read); if (kr != KERN_SUCCESS) { switch (err_get_code(kr)) { @@ -1157,6 +1318,13 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address } return -1; } + if (bytes_read != (mach_vm_size_t)len) { + PyErr_Format(PyExc_OSError, + "mach_vm_read_overwrite read %llu of %zu bytes for PID %d at " + "address 0x%lx", + (unsigned long long)bytes_read, len, handle->pid, remote_address); + return -1; + } return 0; #else Py_UNREACHABLE(); @@ -1168,6 +1336,9 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address static int _Py_RemoteDebug_WriteRemoteMemoryFallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { + if (len == 0) { + return 0; + } if (handle->memfd == -1) { if (open_proc_mem_fd(handle) < 0) { return -1; @@ -1185,10 +1356,19 @@ _Py_RemoteDebug_WriteRemoteMemoryFallback(proc_handle_t *handle, uintptr_t remot written = pwritev(handle->memfd, local, 1, offset); if (written < 0) { + int err = errno; + errno = err; PyErr_SetFromErrno(PyExc_OSError); return -1; } + if (written == 0) { + PyErr_Format(PyExc_OSError, + "pwritev wrote 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial write %zd bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += written; } while ((size_t)written != local[0].iov_len); return 0; @@ -1199,19 +1379,29 @@ _Py_RemoteDebug_WriteRemoteMemoryFallback(proc_handle_t *handle, uintptr_t remot UNUSED static int _Py_RemoteDebug_WriteRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { + if (len == 0) { + return 0; + } #ifdef MS_WINDOWS SIZE_T written = 0; SIZE_T result = 0; do { if (!WriteProcessMemory(handle->hProcess, (LPVOID)(remote_address + result), (const char*)src + result, len - result, &written)) { - PyErr_SetFromWindowsErr(0); DWORD error = GetLastError(); + PyErr_SetFromWindowsErr(error); _set_debug_exception_cause(PyExc_OSError, "WriteProcessMemory failed for PID %d at address 0x%lx " "(size %zu, partial write %zu bytes): Windows error %lu", handle->pid, remote_address + result, len - result, result, error); return -1; } + if (written == 0) { + PyErr_Format(PyExc_OSError, + "WriteProcessMemory wrote 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial write %zu bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += written; } while (result < len); return 0; @@ -1232,17 +1422,26 @@ _Py_RemoteDebug_WriteRemoteMemory(proc_handle_t *handle, uintptr_t remote_addres written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); if (written < 0) { - if (errno == ENOSYS) { + int err = errno; + if (err == ENOSYS) { return _Py_RemoteDebug_WriteRemoteMemoryFallback(handle, remote_address, len, src); } + errno = err; PyErr_SetFromErrno(PyExc_OSError); _set_debug_exception_cause(PyExc_OSError, "process_vm_writev failed for PID %d at address 0x%lx " "(size %zu, partial write %zd bytes): %s", - handle->pid, remote_address + result, len - result, result, strerror(errno)); + handle->pid, remote_address + result, len - result, result, strerror(err)); return -1; } + if (written == 0) { + PyErr_Format(PyExc_OSError, + "process_vm_writev wrote 0 bytes for PID %d at address 0x%lx " + "(size %zu, partial write %zd bytes)", + handle->pid, remote_address + result, len - result, result); + return -1; + } result += written; } while ((size_t)written != local[0].iov_len); return 0; @@ -1286,8 +1485,9 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } - // Search for valid cached page - for (int i = 0; i < MAX_PAGES; i++) { + // Search only the pages used since the last clear. The cache is cleared + // between profiler samples, so entries are packed at the front. + for (int i = 0; i < handle->page_cache_count; i++) { page_cache_entry_t *entry = &handle->pages[i]; if (entry->valid && entry->page_addr == page_base) { memcpy(out, entry->data + offset_in_page, size); @@ -1295,33 +1495,31 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, } } - // Find reusable slot - for (int i = 0; i < MAX_PAGES; i++) { - page_cache_entry_t *entry = &handle->pages[i]; - if (!entry->valid) { + if (handle->page_cache_count < MAX_PAGES) { + page_cache_entry_t *entry = &handle->pages[handle->page_cache_count]; + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); if (entry->data == NULL) { - entry->data = PyMem_RawMalloc(page_size); - if (entry->data == NULL) { - PyErr_NoMemory(); - _set_debug_exception_cause(PyExc_MemoryError, - "Cannot allocate %zu bytes for page cache entry " - "during read from PID %d at address 0x%lx", - page_size, handle->pid, addr); - return -1; - } - } - - if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { - // Try to just copy the exact amount as a fallback - PyErr_Clear(); - goto fallback; + PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate %zu bytes for page cache entry " + "during read from PID %d at address 0x%lx", + page_size, handle->pid, addr); + return -1; } + } - entry->page_addr = page_base; - entry->valid = 1; - memcpy(out, entry->data + offset_in_page, size); - return 0; + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact amount as a fallback + PyErr_Clear(); + goto fallback; } + + entry->page_addr = page_base; + entry->valid = 1; + handle->page_cache_count++; + memcpy(out, entry->data + offset_in_page, size); + return 0; } fallback: @@ -1329,6 +1527,49 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } +typedef struct { + uintptr_t remote_addr; + void *local_buf; + size_t size; +} _Py_RemoteReadSegment; + +#define _PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS 4 + +// Batched read of multiple remote regions in a single syscall when supported. +// Returns total bytes read (>= 0) on success, -1 if batched reads are +// unavailable or the syscall failed. Callers compare the return value against +// cumulative segment sizes to determine which segments were fully populated. +UNUSED static Py_ssize_t +_Py_RemoteDebug_BatchedReadRemoteMemory( + proc_handle_t *handle, + const _Py_RemoteReadSegment *segments, + int nsegs) +{ +#if defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd == -1 + && nsegs > 0 + && nsegs <= _PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS) { + struct iovec local[_PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS]; + struct iovec remote[_PY_REMOTE_DEBUG_MAX_BATCHED_SEGMENTS]; + for (int i = 0; i < nsegs; i++) { + local[i].iov_base = segments[i].local_buf; + local[i].iov_len = segments[i].size; + remote[i].iov_base = (void *)segments[i].remote_addr; + remote[i].iov_len = segments[i].size; + } + ssize_t nread = process_vm_readv(handle->pid, local, nsegs, remote, nsegs, 0); + if (nread >= 0) { + return (Py_ssize_t)nread; + } + } +#else + (void)handle; + (void)segments; + (void)nsegs; +#endif + return -1; +} + UNUSED static int _Py_RemoteDebug_ReadDebugOffsets( proc_handle_t *handle, diff --git a/Python/slots.c b/Python/slots.c new file mode 100644 index 000000000000000..6bc74c727230e0e --- /dev/null +++ b/Python/slots.c @@ -0,0 +1,404 @@ +/* Common handling of type/module slots + */ + +#include "Python.h" + +#include "pycore_slots.h" + +#include <stdio.h> + +// Iterating through a recursive structure doesn't look great in a debugger. +// Flip the #if to 1 to get a trace on stderr. +// (The messages can also serve as code comments.) +#if 0 +#define MSG(...) { \ + fprintf(stderr, "slotiter: " __VA_ARGS__); fprintf(stderr, "\n");} +#else +#define MSG(...) +#endif + +static char* +kind_name(_PySlot_KIND kind) +{ + switch (kind) { + case _PySlot_KIND_TYPE: return "type"; + case _PySlot_KIND_MOD: return "module"; + case _PySlot_KIND_COMPAT: return "compat"; + case _PySlot_KIND_SLOT: return "generic slot"; + } + Py_UNREACHABLE(); +} + +static void +init_with_kind(_PySlotIterator *it, const void *slots, + _PySlot_KIND result_kind, + _PySlot_KIND slot_struct_kind) +{ + MSG(""); + MSG("init (%s slot iterator)", kind_name(result_kind)); + it->state = it->states; + it->state->any_slot = slots; + it->state->slot_struct_kind = slot_struct_kind; + it->kind = result_kind; + it->name = NULL; + it->recursion_level = 0; + it->is_at_end = false; + it->is_first_run = true; + it->current.sl_id = 0; + memset(it->seen, 0, sizeof(it->seen)); +} + +void +_PySlotIterator_Init(_PySlotIterator *it, const PySlot *slots, + _PySlot_KIND result_kind) +{ + init_with_kind(it, slots, result_kind, _PySlot_KIND_SLOT); +} + +void +_PySlotIterator_InitLegacy(_PySlotIterator *it, const void *slots, + _PySlot_KIND kind) +{ + init_with_kind(it, slots, kind, kind); +} + +void +_PySlotIterator_Rewind(_PySlotIterator *it, const void *slots) +{ + MSG(""); + MSG("rewind (%s slot iterator)", kind_name(it->kind)); + assert (it->is_at_end); + assert (it->recursion_level == 0); + assert (it->state == it->states); + it->is_at_end = false; + it->state->any_slot = slots; + it->is_first_run = false; +} + +static Py_ssize_t +seen_index(uint16_t id) +{ + return id / _PySlot_SEEN_ENTRY_BITS; +} + +static unsigned int +seen_mask(uint16_t id) +{ + return ((unsigned int)1) << (id % _PySlot_SEEN_ENTRY_BITS); +} + +bool +_PySlotIterator_SawSlot(_PySlotIterator *it, int id) +{ + assert (id > 0); + assert (id < _Py_slot_COUNT); + return it->seen[seen_index(id)] & seen_mask(id); +} + +// Advance `it` to the next entry. Currently cannot fail. +static void +advance(_PySlotIterator *it) +{ + MSG("advance (at level %d)", (int)it->recursion_level); + switch (it->state->slot_struct_kind) { + case _PySlot_KIND_SLOT: it->state->slot++; break; + case _PySlot_KIND_TYPE: it->state->tp_slot++; break; + case _PySlot_KIND_MOD: it->state->mod_slot++; break; + default: + Py_UNREACHABLE(); + } +} + +static int handle_first_run(_PySlotIterator *it); + +bool +_PySlotIterator_Next(_PySlotIterator *it) +{ + MSG("next"); + assert(it); + assert(!it->is_at_end); + assert(!PyErr_Occurred()); + + it->current.sl_id = -1; + + while (true) { + if (it->state->slot == NULL) { + if (it->recursion_level == 0) { + MSG("end (initial nesting level done)"); + it->is_at_end = true; + return 0; + } + MSG("pop nesting level %d", (int)it->recursion_level); + it->recursion_level--; + it->state = &it->states[it->recursion_level]; + advance(it); + continue; + } + + switch (it->state->slot_struct_kind) { + case _PySlot_KIND_SLOT: { + MSG("copying PySlot structure"); + it->current = *it->state->slot; + } break; + case _PySlot_KIND_TYPE: { + MSG("converting PyType_Slot structure"); + memset(&it->current, 0, sizeof(it->current)); + it->current.sl_id = (uint16_t)it->state->tp_slot->slot; + it->current.sl_flags = PySlot_INTPTR; + it->current.sl_ptr = (void*)it->state->tp_slot->pfunc; + } break; + case _PySlot_KIND_MOD: { + MSG("converting PyModuleDef_Slot structure"); + memset(&it->current, 0, sizeof(it->current)); + it->current.sl_id = (uint16_t)it->state->mod_slot->slot; + it->current.sl_flags = PySlot_INTPTR; + it->current.sl_ptr = (void*)it->state->mod_slot->value; + } break; + default: { + Py_UNREACHABLE(); + } break; + } + + /* shorter local names */ + PySlot *const result = &it->current; + uint16_t flags = result->sl_flags; + + MSG("slot %d, flags 0x%x, from %p", + (int)result->sl_id, (unsigned)flags, it->state->slot); + + uint16_t orig_id = result->sl_id; + switch (it->kind) { + case _PySlot_KIND_TYPE: + result->sl_id = _PySlot_resolve_type_slot(result->sl_id); + break; + case _PySlot_KIND_MOD: + result->sl_id = _PySlot_resolve_mod_slot(result->sl_id); + break; + default: + Py_UNREACHABLE(); + } + MSG("resolved to slot %d (%s)", + (int)result->sl_id, _PySlot_GetName(result->sl_id)); + + if (result->sl_id == Py_slot_invalid) { + MSG("error (unknown/invalid slot)"); + if (flags & PySlot_OPTIONAL) { + advance(it); + continue; + } + _PySlot_err_bad_slot(kind_name(it->kind), orig_id); + goto error; + } + if (result->sl_id == Py_slot_end) { + MSG("sentinel slot, flags %x", (unsigned)flags); + if (flags & PySlot_OPTIONAL) { + MSG("error (bad flags on sentinel)"); + PyErr_Format(PyExc_SystemError, + "invalid flags for Py_slot_end: 0x%x", + (unsigned int)flags); + goto error; + } + it->state->slot = NULL; + continue; + } + + if (result->sl_id == Py_slot_subslots + || result->sl_id == Py_tp_slots + || result->sl_id == Py_mod_slots + ) { + if (result->sl_ptr == NULL) { + MSG("NULL subslots; skipping"); + advance(it); + continue; + } + if ((it->states[0].slot_struct_kind == _PySlot_KIND_MOD) + && (it->state->slot_struct_kind == _PySlot_KIND_SLOT) + && !(result->sl_flags & PySlot_STATIC)) + { + PyErr_Format(PyExc_SystemError, + "slots included from PyModuleDef must be static"); + goto error; + } + it->recursion_level++; + MSG("recursing into level %d", it->recursion_level); + if (it->recursion_level >= _PySlot_MAX_NESTING) { + MSG("error (too much nesting)"); + PyErr_Format(PyExc_SystemError, + "%s (slot %d): too many levels of nested slots", + _PySlot_GetName(result->sl_id), orig_id); + goto error; + } + it->state = &it->states[it->recursion_level]; + memset(it->state, 0, sizeof(_PySlotIterator_state)); + it->state->slot = result->sl_ptr; + switch (result->sl_id) { + case Py_slot_subslots: + it->state->slot_struct_kind = _PySlot_KIND_SLOT; break; + case Py_tp_slots: + it->state->slot_struct_kind = _PySlot_KIND_TYPE; break; + case Py_mod_slots: + it->state->slot_struct_kind = _PySlot_KIND_MOD; break; + } + continue; + } + + if (flags & PySlot_INTPTR) { + MSG("casting from intptr"); + /* this should compile to nothing on common architectures */ + switch (_PySlot_get_dtype(result->sl_id)) { + case _PySlot_DTYPE_SIZE: { + result->sl_size = (Py_ssize_t)(intptr_t)result->sl_ptr; + } break; + case _PySlot_DTYPE_INT64: { + result->sl_int64 = (int64_t)(intptr_t)result->sl_ptr; + } break; + case _PySlot_DTYPE_UINT64: { + result->sl_uint64 = (uint64_t)(intptr_t)result->sl_ptr; + } break; + case _PySlot_DTYPE_PTR: + case _PySlot_DTYPE_FUNC: + case _PySlot_DTYPE_VOID: + break; + } + } + + advance(it); + switch (_PySlot_get_dtype(result->sl_id)) { + case _PySlot_DTYPE_VOID: + case _PySlot_DTYPE_PTR: + MSG("result: %d (%s): %p", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (void*)result->sl_ptr); + break; + case _PySlot_DTYPE_FUNC: + MSG("result: %d (%s): %p", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (void*)result->sl_func); + break; + case _PySlot_DTYPE_SIZE: + MSG("result: %d (%s): %zd", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (Py_ssize_t)result->sl_size); + break; + case _PySlot_DTYPE_INT64: + MSG("result: %d (%s): %ld", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (long)result->sl_int64); + break; + case _PySlot_DTYPE_UINT64: + MSG("result: %d (%s): %lu (0x%lx)", + (int)result->sl_id, _PySlot_GetName(result->sl_id), + (unsigned long)result->sl_int64, + (unsigned long)result->sl_int64); + break; + } + assert (result->sl_id > 0); + assert (result->sl_id <= _Py_slot_COUNT); + if (it->is_first_run && (handle_first_run(it) < 0)) { + goto error; + } + return result->sl_id != Py_slot_end; + } + Py_UNREACHABLE(); + +error: + it->current.sl_id = Py_slot_invalid; + return true; +} + +/* Validate current slot, and do bookkeeping */ +static int +handle_first_run(_PySlotIterator *it) +{ + int id = it->current.sl_id; + + if (_PySlot_get_must_be_static(id)) { + if (!(it->current.sl_flags & PySlot_STATIC) + && (it->state->slot_struct_kind == _PySlot_KIND_SLOT)) + { + PyErr_Format( + PyExc_SystemError, + "%s requires PySlot_STATIC", + _PySlot_GetName(id)); + return -1; + } + } + + _PySlot_PROBLEM_HANDLING null_handling = _PySlot_get_null_handling(id); + if (null_handling != _PySlot_PROBLEM_ALLOW) { + bool is_null = false; + switch (_PySlot_get_dtype(id)) { + case _PySlot_DTYPE_PTR: { + is_null = it->current.sl_ptr == NULL; + } break; + case _PySlot_DTYPE_FUNC: { + is_null = it->current.sl_func == NULL; + } break; + default: { + //Py_UNREACHABLE(); + } break; + } + if (is_null) { + MSG("slot is NULL but shouldn't"); + if (null_handling == _PySlot_PROBLEM_REJECT) { + MSG("error (NULL rejected)"); + PyErr_Format(PyExc_SystemError, + "NULL not allowed for slot %s", + _PySlot_GetName(id)); + return -1; + } + if (it->states[0].slot_struct_kind == _PySlot_KIND_SLOT) { + MSG("deprecated NULL"); + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, + 1, + "NULL value in slot %s is deprecated", + _PySlot_GetName(id)) < 0) + { + return -1; + } + } + else { + MSG("unwanted NULL in legacy struct"); + } + } + } + + _PySlot_PROBLEM_HANDLING duplicate_handling = _PySlot_get_duplicate_handling(id); + if (duplicate_handling != _PySlot_PROBLEM_ALLOW) { + if (_PySlotIterator_SawSlot(it, id)) { + MSG("slot was seen before but shouldn't be duplicated"); + if (duplicate_handling == _PySlot_PROBLEM_REJECT) { + MSG("error (duplicate rejected)"); + PyErr_Format( + PyExc_SystemError, + "%s%s%s has multiple %s (%d) slots", + kind_name(it->kind), + it->name ? " " : "", + it->name ? it->name : "", + _PySlot_GetName(id), + (int)it->current.sl_id); + return -1; + } + if (it->states[0].slot_struct_kind == _PySlot_KIND_SLOT) { + MSG("deprecated duplicate"); + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, + 0, + "%s%s%s has multiple %s (%d) slots. This is deprecated.", + kind_name(it->kind), + it->name ? " " : "", + it->name ? it->name : "", + _PySlot_GetName(id), + (int)it->current.sl_id) < 0) { + return -1; + } + } + else { + MSG("unwanted duplicate in legacy struct"); + } + } + } + it->seen[seen_index(id)] |= seen_mask(id); + return 0; +} diff --git a/Python/slots.toml b/Python/slots.toml new file mode 100644 index 000000000000000..626a44d2f2c80e8 --- /dev/null +++ b/Python/slots.toml @@ -0,0 +1,836 @@ +# This file lists all PySlot values +# This should only be used as input to Tools/build/generate_slots.py, +# its format can change at any time (e.g. we can switch to slots.csv) + +# Entries: +# name: name of the slot +# kind: +# - 'type', 'mod': slots to create a particular kind of object +# - 'slot': special slots applicable to any kind of object +# - 'compat': old IDs that need to be resolved +# dtype: data type (tag for the union of sl_ptr, sl_size, etc.) +# equivalents: for 'compat' slots; the slots to resolve to +# is_type_field: slot that corresponds to a field in the type object (or in +# an array like PyNumberMethods). +# functype: C function type, where needed +# duplicates, nulls: How to handle common "problems" -- duplicate slots with +# the same ID, and NULL pointers, respectively +# - 'allow': not a problem for this slot +# - 'deprecated': issue a deprecation warning. Don't use for new slots. +# (typically, the problem was disallowed in docs, but allowed in practice) +# - 'reject': raise error +# The default for duplicate slots is 'reject' +# The default for NULLs is 'reject' for pointer slots; 'allow' for +# non-pointer ones +# must_be_static: true if slot needs the PySlot_STATIC flag (in PySlot struct) + + +[0] +name = 'Py_slot_end' +kind = 'slot' +dtype = 'void' + +[1] +kind = 'compat' +equivalents = {type='Py_bf_getbuffer', mod='Py_mod_create'} + +[2] +kind = 'compat' +equivalents = {type='Py_bf_releasebuffer', mod='Py_mod_exec'} + +[3] +kind = 'compat' +equivalents = {type='Py_mp_ass_subscript', mod='Py_mod_multiple_interpreters'} + +[4] +kind = 'compat' +equivalents = {type='Py_mp_length', mod='Py_mod_gil'} + +[5] +name = 'Py_mp_subscript' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[6] +name = 'Py_nb_absolute' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[7] +name = 'Py_nb_add' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[8] +name = 'Py_nb_and' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[9] +name = 'Py_nb_bool' +kind = 'type' +is_type_field = true +functype = 'inquiry' +duplicates = 'deprecated' +nulls = 'deprecated' + +[10] +name = 'Py_nb_divmod' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[11] +name = 'Py_nb_float' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[12] +name = 'Py_nb_floor_divide' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[13] +name = 'Py_nb_index' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[14] +name = 'Py_nb_inplace_add' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[15] +name = 'Py_nb_inplace_and' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[16] +name = 'Py_nb_inplace_floor_divide' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[17] +name = 'Py_nb_inplace_lshift' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[18] +name = 'Py_nb_inplace_multiply' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[19] +name = 'Py_nb_inplace_or' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[20] +name = 'Py_nb_inplace_power' +kind = 'type' +is_type_field = true +functype = 'ternaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[21] +name = 'Py_nb_inplace_remainder' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[22] +name = 'Py_nb_inplace_rshift' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[23] +name = 'Py_nb_inplace_subtract' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[24] +name = 'Py_nb_inplace_true_divide' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[25] +name = 'Py_nb_inplace_xor' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[26] +name = 'Py_nb_int' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[27] +name = 'Py_nb_invert' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[28] +name = 'Py_nb_lshift' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[29] +name = 'Py_nb_multiply' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[30] +name = 'Py_nb_negative' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[31] +name = 'Py_nb_or' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[32] +name = 'Py_nb_positive' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[33] +name = 'Py_nb_power' +kind = 'type' +is_type_field = true +functype = 'ternaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[34] +name = 'Py_nb_remainder' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[35] +name = 'Py_nb_rshift' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[36] +name = 'Py_nb_subtract' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[37] +name = 'Py_nb_true_divide' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[38] +name = 'Py_nb_xor' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[39] +name = 'Py_sq_ass_item' +kind = 'type' +is_type_field = true +functype = 'ssizeobjargproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[40] +name = 'Py_sq_concat' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[41] +name = 'Py_sq_contains' +kind = 'type' +is_type_field = true +functype = 'objobjproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[42] +name = 'Py_sq_inplace_concat' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[43] +name = 'Py_sq_inplace_repeat' +kind = 'type' +is_type_field = true +functype = 'ssizeargfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[44] +name = 'Py_sq_item' +kind = 'type' +is_type_field = true +functype = 'ssizeargfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[45] +name = 'Py_sq_length' +kind = 'type' +is_type_field = true +functype = 'lenfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[46] +name = 'Py_sq_repeat' +kind = 'type' +is_type_field = true +functype = 'ssizeargfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[47] +name = 'Py_tp_alloc' +kind = 'type' +is_type_field = true +functype = 'allocfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[48] +name = 'Py_tp_base' +kind = 'type' +is_type_field = true +dtype = 'ptr' +duplicates = 'deprecated' +nulls = 'deprecated' + +[49] +name = 'Py_tp_bases' +kind = 'type' +is_type_field = true +dtype = 'ptr' +duplicates = 'deprecated' +nulls = 'deprecated' + +[50] +name = 'Py_tp_call' +kind = 'type' +is_type_field = true +functype = 'ternaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[51] +name = 'Py_tp_clear' +kind = 'type' +is_type_field = true +functype = 'inquiry' +duplicates = 'deprecated' +nulls = 'deprecated' + +[52] +name = 'Py_tp_dealloc' +kind = 'type' +is_type_field = true +functype = 'destructor' +duplicates = 'deprecated' +nulls = 'deprecated' + +[53] +name = 'Py_tp_del' +kind = 'type' +is_type_field = true +functype = 'destructor' +duplicates = 'deprecated' +nulls = 'deprecated' + +[54] +name = 'Py_tp_descr_get' +kind = 'type' +is_type_field = true +functype = 'descrgetfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[55] +name = 'Py_tp_descr_set' +kind = 'type' +is_type_field = true +functype = 'descrsetfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[56] +name = 'Py_tp_doc' +kind = 'type' +is_type_field = true +dtype = 'ptr' +nulls = 'allow' + +[57] +name = 'Py_tp_getattr' +kind = 'type' +is_type_field = true +functype = 'getattrfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[58] +name = 'Py_tp_getattro' +kind = 'type' +is_type_field = true +functype = 'getattrofunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[59] +name = 'Py_tp_hash' +kind = 'type' +is_type_field = true +functype = 'hashfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[60] +name = 'Py_tp_init' +kind = 'type' +is_type_field = true +functype = 'initproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[61] +name = 'Py_tp_is_gc' +kind = 'type' +is_type_field = true +functype = 'inquiry' +duplicates = 'deprecated' +nulls = 'deprecated' + +[62] +name = 'Py_tp_iter' +kind = 'type' +is_type_field = true +functype = 'getiterfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[63] +name = 'Py_tp_iternext' +kind = 'type' +is_type_field = true +functype = 'iternextfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[64] +name = 'Py_tp_methods' +kind = 'type' +is_type_field = true +dtype = 'ptr' +duplicates = 'deprecated' +nulls = 'deprecated' +must_be_static = true + +[65] +name = 'Py_tp_new' +kind = 'type' +is_type_field = true +functype = 'newfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[66] +name = 'Py_tp_repr' +kind = 'type' +is_type_field = true +functype = 'reprfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[67] +name = 'Py_tp_richcompare' +kind = 'type' +is_type_field = true +functype = 'richcmpfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[68] +name = 'Py_tp_setattr' +kind = 'type' +is_type_field = true +functype = 'setattrfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[69] +name = 'Py_tp_setattro' +kind = 'type' +is_type_field = true +functype = 'setattrofunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[70] +name = 'Py_tp_str' +kind = 'type' +is_type_field = true +functype = 'reprfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[71] +name = 'Py_tp_traverse' +kind = 'type' +is_type_field = true +functype = 'traverseproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[72] +name = 'Py_tp_members' +kind = 'type' +is_type_field = true +dtype = 'ptr' +nulls = 'reject' +must_be_static = true + +[73] +name = 'Py_tp_getset' +kind = 'type' +is_type_field = true +dtype = 'ptr' +duplicates = 'deprecated' +nulls = 'deprecated' +must_be_static = true + +[74] +name = 'Py_tp_free' +kind = 'type' +is_type_field = true +functype = 'freefunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[75] +name = 'Py_nb_matrix_multiply' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[76] +name = 'Py_nb_inplace_matrix_multiply' +kind = 'type' +is_type_field = true +functype = 'binaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[77] +name = 'Py_am_await' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[78] +name = 'Py_am_aiter' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[79] +name = 'Py_am_anext' +kind = 'type' +is_type_field = true +functype = 'unaryfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[80] +name = 'Py_tp_finalize' +kind = 'type' +is_type_field = true +functype = 'destructor' +duplicates = 'deprecated' +nulls = 'deprecated' + +[81] +name = 'Py_am_send' +kind = 'type' +is_type_field = true +functype = 'sendfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[82] +name = 'Py_tp_vectorcall' +kind = 'type' +is_type_field = true +functype = 'vectorcallfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[83] +name = 'Py_tp_token' +kind = 'type' +is_type_field = true +dtype = 'ptr' +field = 'ht_token' +duplicates = 'deprecated' +nulls = 'allow' + +[84] +name = 'Py_mod_create' +kind = 'mod' +dtype = 'func' +nulls = 'deprecated' + +[85] +name = 'Py_mod_exec' +kind = 'mod' +dtype = 'func' +duplicates = 'allow' # only alowed in PyModuleDef.m_slots +nulls = 'reject' + +[86] +name = 'Py_mod_multiple_interpreters' +kind = 'mod' +dtype = 'uint64' + +[87] +name = 'Py_mod_gil' +kind = 'mod' +dtype = 'uint64' + +[88] +name = 'Py_bf_getbuffer' +kind = 'type' +is_type_field = true +functype = 'getbufferproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[89] +name = 'Py_bf_releasebuffer' +kind = 'type' +is_type_field = true +functype = 'releasebufferproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[90] +name = 'Py_mp_ass_subscript' +kind = 'type' +is_type_field = true +functype = 'objobjargproc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[91] +name = 'Py_mp_length' +kind = 'type' +is_type_field = true +functype = 'lenfunc' +duplicates = 'deprecated' +nulls = 'deprecated' + +[92] +name = 'Py_slot_subslots' +kind = 'slot' +dtype = 'ptr' +nulls = 'allow' + +[93] +name = 'Py_tp_slots' +kind = 'type' +dtype = 'ptr' +nulls = 'allow' + +[94] +name = 'Py_mod_slots' +kind = 'mod' +dtype = 'ptr' +nulls = 'allow' + +[95] +name = 'Py_tp_name' +kind = 'type' +dtype = 'ptr' + +[96] +name = 'Py_tp_basicsize' +kind = 'type' +dtype = 'size' + +[97] +name = 'Py_tp_extra_basicsize' +kind = 'type' +dtype = 'size' + +[98] +name = 'Py_tp_itemsize' +kind = 'type' +dtype = 'size' + +[99] +name = 'Py_tp_flags' +kind = 'type' +dtype = 'uint64' + +[100] +name = 'Py_mod_name' +kind = 'mod' +dtype = 'ptr' + +[101] +name = 'Py_mod_doc' +kind = 'mod' +dtype = 'ptr' + +[102] +name = 'Py_mod_state_size' +kind = 'mod' +dtype = 'size' + +[103] +name = 'Py_mod_methods' +kind = 'mod' +dtype = 'ptr' +must_be_static = true + +[104] +name = 'Py_mod_state_traverse' +kind = 'mod' +dtype = 'func' + +[105] +name = 'Py_mod_state_clear' +kind = 'mod' +dtype = 'func' + +[106] +name = 'Py_mod_state_free' +kind = 'mod' +dtype = 'func' + +[107] +name = 'Py_tp_metaclass' +kind = 'type' +dtype = 'ptr' + +[108] +name = 'Py_tp_module' +kind = 'type' +dtype = 'ptr' + +[109] +name = 'Py_mod_abi' +kind = 'mod' +dtype = 'ptr' +duplicates = 'allow' + +[110] +name = 'Py_mod_token' +kind = 'mod' +dtype = 'ptr' diff --git a/Python/slots_generated.c b/Python/slots_generated.c new file mode 100644 index 000000000000000..b8da8fb7890d8c5 --- /dev/null +++ b/Python/slots_generated.c @@ -0,0 +1,119 @@ +/* Generated by Tools/build/generate_slots.py */ + +#include "Python.h" +#include "pycore_slots.h" // _PySlot_names + +const char *const _PySlot_names[] = { + "Py_slot_end", + "Py_bf_getbuffer/Py_mod_create", + "Py_bf_releasebuffer/Py_mod_exec", + "Py_mp_ass_subscript/Py_mod_multiple_interpreters", + "Py_mp_length/Py_mod_gil", + "Py_mp_subscript", + "Py_nb_absolute", + "Py_nb_add", + "Py_nb_and", + "Py_nb_bool", + "Py_nb_divmod", + "Py_nb_float", + "Py_nb_floor_divide", + "Py_nb_index", + "Py_nb_inplace_add", + "Py_nb_inplace_and", + "Py_nb_inplace_floor_divide", + "Py_nb_inplace_lshift", + "Py_nb_inplace_multiply", + "Py_nb_inplace_or", + "Py_nb_inplace_power", + "Py_nb_inplace_remainder", + "Py_nb_inplace_rshift", + "Py_nb_inplace_subtract", + "Py_nb_inplace_true_divide", + "Py_nb_inplace_xor", + "Py_nb_int", + "Py_nb_invert", + "Py_nb_lshift", + "Py_nb_multiply", + "Py_nb_negative", + "Py_nb_or", + "Py_nb_positive", + "Py_nb_power", + "Py_nb_remainder", + "Py_nb_rshift", + "Py_nb_subtract", + "Py_nb_true_divide", + "Py_nb_xor", + "Py_sq_ass_item", + "Py_sq_concat", + "Py_sq_contains", + "Py_sq_inplace_concat", + "Py_sq_inplace_repeat", + "Py_sq_item", + "Py_sq_length", + "Py_sq_repeat", + "Py_tp_alloc", + "Py_tp_base", + "Py_tp_bases", + "Py_tp_call", + "Py_tp_clear", + "Py_tp_dealloc", + "Py_tp_del", + "Py_tp_descr_get", + "Py_tp_descr_set", + "Py_tp_doc", + "Py_tp_getattr", + "Py_tp_getattro", + "Py_tp_hash", + "Py_tp_init", + "Py_tp_is_gc", + "Py_tp_iter", + "Py_tp_iternext", + "Py_tp_methods", + "Py_tp_new", + "Py_tp_repr", + "Py_tp_richcompare", + "Py_tp_setattr", + "Py_tp_setattro", + "Py_tp_str", + "Py_tp_traverse", + "Py_tp_members", + "Py_tp_getset", + "Py_tp_free", + "Py_nb_matrix_multiply", + "Py_nb_inplace_matrix_multiply", + "Py_am_await", + "Py_am_aiter", + "Py_am_anext", + "Py_tp_finalize", + "Py_am_send", + "Py_tp_vectorcall", + "Py_tp_token", + "Py_mod_create", + "Py_mod_exec", + "Py_mod_multiple_interpreters", + "Py_mod_gil", + "Py_bf_getbuffer", + "Py_bf_releasebuffer", + "Py_mp_ass_subscript", + "Py_mp_length", + "Py_slot_subslots", + "Py_tp_slots", + "Py_mod_slots", + "Py_tp_name", + "Py_tp_basicsize", + "Py_tp_extra_basicsize", + "Py_tp_itemsize", + "Py_tp_flags", + "Py_mod_name", + "Py_mod_doc", + "Py_mod_state_size", + "Py_mod_methods", + "Py_mod_state_traverse", + "Py_mod_state_clear", + "Py_mod_state_free", + "Py_tp_metaclass", + "Py_tp_module", + "Py_mod_abi", + "Py_mod_token", + NULL +}; diff --git a/Python/specialize.c b/Python/specialize.c index b50728d4a2f3f3a..eca0c4ac2177699 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -113,7 +113,8 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters } #else for (Py_ssize_t i = 0; i < size-1; i++) { - if (instructions[i].op.code == GET_ITER) { + int opcode = instructions[i].op.code; + if (opcode == GET_ITER) { fixup_getiter(&instructions[i], flags); } i += _PyOpcode_Caches[opcode]; @@ -1285,7 +1286,6 @@ specialize_attr_loadclassattr(PyObject *owner, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_VERSIONS); return 0; } - write_u32(cache->keys_version, shared_keys_version); specialize(instr, is_method ? LOAD_ATTR_METHOD_WITH_VALUES : LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); } else { @@ -2124,55 +2124,23 @@ is_compactlong(PyObject *v) _PyLong_IsCompact((PyLongObject *)v); } -/* sequence * int helpers: bypass PyNumber_Multiply dispatch overhead - by calling sq_repeat directly with PyLong_AsSsize_t. */ - -static inline PyObject * -seq_int_multiply(PyObject *seq, PyObject *n, - ssizeargfunc repeat) -{ - Py_ssize_t count = PyLong_AsSsize_t(n); - if (count == -1 && PyErr_Occurred()) { - return NULL; - } - return repeat(seq, count); -} - -static PyObject * -str_int_multiply(PyObject *lhs, PyObject *rhs) -{ - return seq_int_multiply(lhs, rhs, PyUnicode_Type.tp_as_sequence->sq_repeat); -} - -static PyObject * -int_str_multiply(PyObject *lhs, PyObject *rhs) -{ - return seq_int_multiply(rhs, lhs, PyUnicode_Type.tp_as_sequence->sq_repeat); -} - -static PyObject * -bytes_int_multiply(PyObject *lhs, PyObject *rhs) -{ - return seq_int_multiply(lhs, rhs, PyBytes_Type.tp_as_sequence->sq_repeat); -} - -static PyObject * -int_bytes_multiply(PyObject *lhs, PyObject *rhs) -{ - return seq_int_multiply(rhs, lhs, PyBytes_Type.tp_as_sequence->sq_repeat); -} - -static PyObject * -tuple_int_multiply(PyObject *lhs, PyObject *rhs) -{ - return seq_int_multiply(lhs, rhs, PyTuple_Type.tp_as_sequence->sq_repeat); -} - -static PyObject * -int_tuple_multiply(PyObject *lhs, PyObject *rhs) -{ - return seq_int_multiply(rhs, lhs, PyTuple_Type.tp_as_sequence->sq_repeat); -} +#define SEQ_INT_MULTIPLY_ACTION(NAME, REPEAT, SEQ, COUNT) \ + static PyObject * \ + (NAME)(PyObject *lhs, PyObject *rhs) \ + { \ + Py_ssize_t count = PyLong_AsSsize_t(COUNT); \ + if (count == -1 && PyErr_Occurred()) { \ + return NULL; \ + } \ + return REPEAT(SEQ, count); \ + } +SEQ_INT_MULTIPLY_ACTION(str_int_multiply, _PyUnicode_Repeat, lhs, rhs) +SEQ_INT_MULTIPLY_ACTION(int_str_multiply, _PyUnicode_Repeat, rhs, lhs) +SEQ_INT_MULTIPLY_ACTION(bytes_int_multiply, _PyBytes_Repeat, lhs, rhs) +SEQ_INT_MULTIPLY_ACTION(int_bytes_multiply, _PyBytes_Repeat, rhs, lhs) +SEQ_INT_MULTIPLY_ACTION(tuple_int_multiply, _PyTuple_Repeat, lhs, rhs) +SEQ_INT_MULTIPLY_ACTION(int_tuple_multiply, _PyTuple_Repeat, rhs, lhs) +#undef SEQ_INT_MULTIPLY_ACTION static int compactlongs_guard(PyObject *lhs, PyObject *rhs) @@ -2278,12 +2246,20 @@ static _PyBinaryOpSpecializationDescr binaryop_extend_descrs[] = { {NB_SUBTRACT, float_compactlong_guard, float_compactlong_subtract, &PyFloat_Type, 1, NULL, NULL}, {NB_TRUE_DIVIDE, nonzero_float_compactlong_guard, float_compactlong_true_div, &PyFloat_Type, 1, NULL, NULL}, {NB_MULTIPLY, float_compactlong_guard, float_compactlong_multiply, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_ADD, float_compactlong_guard, float_compactlong_add, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_SUBTRACT, float_compactlong_guard, float_compactlong_subtract, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_TRUE_DIVIDE, nonzero_float_compactlong_guard, float_compactlong_true_div, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_MULTIPLY, float_compactlong_guard, float_compactlong_multiply, &PyFloat_Type, 1, NULL, NULL}, /* long-float arithmetic: guards also check NaN and compactness. */ {NB_ADD, compactlong_float_guard, compactlong_float_add, &PyFloat_Type, 1, NULL, NULL}, {NB_SUBTRACT, compactlong_float_guard, compactlong_float_subtract, &PyFloat_Type, 1, NULL, NULL}, {NB_TRUE_DIVIDE, nonzero_compactlong_float_guard, compactlong_float_true_div, &PyFloat_Type, 1, NULL, NULL}, {NB_MULTIPLY, compactlong_float_guard, compactlong_float_multiply, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_ADD, compactlong_float_guard, compactlong_float_add, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_SUBTRACT, compactlong_float_guard, compactlong_float_subtract, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_TRUE_DIVIDE, nonzero_compactlong_float_guard, compactlong_float_true_div, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_MULTIPLY, compactlong_float_guard, compactlong_float_multiply, &PyFloat_Type, 1, NULL, NULL}, /* list-list concatenation: _PyList_Concat always allocates a new list */ {NB_ADD, NULL, _PyList_Concat, &PyList_Type, 1, &PyList_Type, &PyList_Type}, @@ -2292,8 +2268,8 @@ static _PyBinaryOpSpecializationDescr binaryop_extend_descrs[] = { to be a freshly allocated object. */ {NB_ADD, NULL, _PyTuple_Concat, &PyTuple_Type, 0, &PyTuple_Type, &PyTuple_Type}, - /* str * int / int * str: call unicode_repeat directly. - unicode_repeat returns the original when n == 1. */ + /* str * int / int * str: call _PyUnicode_Repeat directly. + _PyUnicode_Repeat returns the original when n == 1. */ {NB_MULTIPLY, NULL, str_int_multiply, &PyUnicode_Type, 0, &PyUnicode_Type, &PyLong_Type}, {NB_MULTIPLY, NULL, int_str_multiply, &PyUnicode_Type, 0, &PyLong_Type, &PyUnicode_Type}, {NB_INPLACE_MULTIPLY, NULL, str_int_multiply, &PyUnicode_Type, 0, &PyUnicode_Type, &PyLong_Type}, @@ -2304,15 +2280,15 @@ static _PyBinaryOpSpecializationDescr binaryop_extend_descrs[] = { {NB_ADD, NULL, _PyBytes_Concat, &PyBytes_Type, 0, &PyBytes_Type, &PyBytes_Type}, {NB_INPLACE_ADD, NULL, _PyBytes_Concat, &PyBytes_Type, 0, &PyBytes_Type, &PyBytes_Type}, - /* bytes * int / int * bytes: call bytes_repeat directly. - bytes_repeat returns the original when n == 1. */ + /* bytes * int / int * bytes: call _PyBytes_Repeat directly. + _PyBytes_Repeat returns the original when n == 1. */ {NB_MULTIPLY, NULL, bytes_int_multiply, &PyBytes_Type, 0, &PyBytes_Type, &PyLong_Type}, {NB_MULTIPLY, NULL, int_bytes_multiply, &PyBytes_Type, 0, &PyLong_Type, &PyBytes_Type}, {NB_INPLACE_MULTIPLY, NULL, bytes_int_multiply, &PyBytes_Type, 0, &PyBytes_Type, &PyLong_Type}, {NB_INPLACE_MULTIPLY, NULL, int_bytes_multiply, &PyBytes_Type, 0, &PyLong_Type, &PyBytes_Type}, - /* tuple * int / int * tuple: call tuple_repeat directly. - tuple_repeat returns the original when n == 1. */ + /* tuple * int / int * tuple: call _PyTuple_Repeat directly. + _PyTuple_Repeat returns the original when n == 1. */ {NB_MULTIPLY, NULL, tuple_int_multiply, &PyTuple_Type, 0, &PyTuple_Type, &PyLong_Type}, {NB_MULTIPLY, NULL, int_tuple_multiply, &PyTuple_Type, 0, &PyLong_Type, &PyTuple_Type}, {NB_INPLACE_MULTIPLY, NULL, tuple_int_multiply, &PyTuple_Type, 0, &PyTuple_Type, &PyLong_Type}, @@ -2761,6 +2737,14 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) specialize(instr, SEND_GEN); return; } + if (tp->_tp_iteritem != NULL) { + specialize(instr, SEND_VIRTUAL); + return; + } + if (tp == &_PyAsyncGenASend_Type) { + specialize(instr, SEND_ASYNC_GEN); + return; + } SPECIALIZATION_FAIL(SEND, _PySpecialization_ClassifyIterator(receiver)); failure: diff --git a/Python/structmember.c b/Python/structmember.c index b88e13ac0462b80..adea8216b8796b5 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -171,19 +171,10 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_SetString(PyExc_AttributeError, "readonly attribute"); return -1; } - if (v == NULL) { - if (l->type == Py_T_OBJECT_EX) { - /* Check if the attribute is set. */ - if (*(PyObject **)addr == NULL) { - PyErr_SetString(PyExc_AttributeError, l->name); - return -1; - } - } - else if (l->type != _Py_T_OBJECT) { - PyErr_SetString(PyExc_TypeError, - "can't delete numeric/char attribute"); - return -1; - } + if (v == NULL && l->type != Py_T_OBJECT_EX && l->type != _Py_T_OBJECT) { + PyErr_SetString(PyExc_TypeError, + "can't delete numeric/char attribute"); + return -1; } switch (l->type) { case Py_T_BOOL:{ @@ -334,6 +325,15 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) oldv = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, Py_XNewRef(v)); Py_END_CRITICAL_SECTION(); + if (v == NULL && oldv == NULL && l->type == Py_T_OBJECT_EX) { + // Raise an exception when attempting to delete an already deleted + // attribute. + // Differently from Py_T_OBJECT_EX, _Py_T_OBJECT does not raise an + // exception here (PyMember_GetOne will return Py_None instead of + // NULL). + PyErr_SetString(PyExc_AttributeError, l->name); + return -1; + } Py_XDECREF(oldv); break; case Py_T_CHAR: { diff --git a/Python/symtable.c b/Python/symtable.c index 2263a2d8db9097d..00ac510fe76b972 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -834,17 +834,22 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, return 0; } // __class__, __classdict__ and __conditional_annotations__ are - // never allowed to be free through a class scope (see - // drop_class_free) + // not allowed to be free through a class scope (see + // drop_class_free) unless children scopes need it if (scope == FREE && ste->ste_type == ClassBlock && (_PyUnicode_EqualToASCIIString(k, "__class__") || _PyUnicode_EqualToASCIIString(k, "__classdict__") || _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { scope = GLOBAL_IMPLICIT; - if (PySet_Discard(comp_free, k) < 0) { + int child_needs_free = is_free_in_any_child(comp, k); + if (child_needs_free < 0) { return 0; } - + if (!child_needs_free) { + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + } if (_PyUnicode_EqualToASCIIString(k, "__class__")) { remove_dunder_class = 1; } @@ -1708,7 +1713,7 @@ symtable_enter_type_param_block(struct symtable *st, identifier name, return 1; } -/* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument. +/* VISIT, VISIT_SEQ and VISIT_SEQ_TAIL take an ASDL type as their second argument. They use the ASDL name to synthesize the name of the C type and the visit function. @@ -2870,6 +2875,7 @@ symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; if (current_type == ClassBlock && !future_annotations) { st->st_cur->ste_can_see_class_scope = 1; + parent_ste->ste_needs_classdict = 1; if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) { return 0; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index c6447d03369a949..d9f7b9c449cfb94 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1790,36 +1790,6 @@ sys_getwindowsversion_impl(PyObject *module) } #pragma warning(pop) - -/*[clinic input] -sys._enablelegacywindowsfsencoding - -Changes the default filesystem encoding to mbcs:replace. - -This is done for consistency with earlier versions of Python. See PEP -529 for more information. - -This is equivalent to defining the PYTHONLEGACYWINDOWSFSENCODING -environment variable before launching Python. -[clinic start generated code]*/ - -static PyObject * -sys__enablelegacywindowsfsencoding_impl(PyObject *module) -/*[clinic end generated code: output=f5c3855b45e24fe9 input=2bfa931a20704492]*/ -{ - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "sys._enablelegacywindowsfsencoding() is deprecated and will be " - "removed in Python 3.16. Use PYTHONLEGACYWINDOWSFSENCODING " - "instead.", 1)) - { - return NULL; - } - if (_PyUnicode_EnableLegacyWindowsFSEncoding() < 0) { - return NULL; - } - Py_RETURN_NONE; -} - #endif /* MS_WINDOWS */ #ifdef HAVE_DLOPEN @@ -1893,6 +1863,7 @@ sys_mdebug_impl(PyObject *module, int flag) /*[clinic input] +@permit_long_summary sys.get_int_max_str_digits Return the maximum string digits limit for non-binary int<->str conversions. @@ -1900,14 +1871,16 @@ Return the maximum string digits limit for non-binary int<->str conversions. static PyObject * sys_get_int_max_str_digits_impl(PyObject *module) -/*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ +/*[clinic end generated code: output=0042f5e8ae0e8631 input=77fb74e987ba7ecb]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->long_state.max_str_digits); + int maxdigits = _Py_atomic_load_int(&interp->long_state.max_str_digits); + return PyLong_FromLong(maxdigits); } /*[clinic input] +@permit_long_summary sys.set_int_max_str_digits maxdigits: int @@ -1917,7 +1890,7 @@ Set the maximum string digits limit for non-binary int<->str conversions. static PyObject * sys_set_int_max_str_digits_impl(PyObject *module, int maxdigits) -/*[clinic end generated code: output=734d4c2511f2a56d input=d7e3f325db6910c5]*/ +/*[clinic end generated code: output=734d4c2511f2a56d input=d4c0bf50c466d57a]*/ { if (_PySys_SetIntMaxStrDigits(maxdigits) < 0) { return NULL; @@ -2129,6 +2102,7 @@ sys__getframe_impl(PyObject *module, int depth) } /*[clinic input] +@permit_long_summary sys._current_frames Return a dict mapping each thread's thread id to its current stack frame. @@ -2138,7 +2112,7 @@ This function should be used for specialized purposes only. static PyObject * sys__current_frames_impl(PyObject *module) -/*[clinic end generated code: output=d2a41ac0a0a3809a input=2a9049c5f5033691]*/ +/*[clinic end generated code: output=d2a41ac0a0a3809a input=e1ce34f43501e0d6]*/ { return _PyThread_CurrentFrames(); } @@ -2317,17 +2291,17 @@ sys__stats_clear_impl(PyObject *module) } /*[clinic input] -@permit_long_docstring_body sys._stats_dump -> bool Dump stats to file, and clears the stats. -Return False if no statistics were not dumped because stats gathering was off. +Return False if no statistics were not dumped because stats gathering +was off. [clinic start generated code]*/ static int sys__stats_dump_impl(PyObject *module) -/*[clinic end generated code: output=6e346b4ba0de4489 input=5a3ab40d2fb5af47]*/ +/*[clinic end generated code: output=6e346b4ba0de4489 input=7f3b7758cb59d2ff]*/ { int res = _Py_PrintSpecializationStats(1); _Py_StatsClear(); @@ -2471,16 +2445,16 @@ sys.remote_exec Executes a file containing Python code in a given remote Python process. This function returns immediately, and the code will be executed by the -target process's main thread at the next available opportunity, similarly -to how signals are handled. There is no interface to determine when the -code has been executed. The caller is responsible for making sure that -the file still exists whenever the remote process tries to read it and that -it hasn't been overwritten. +target process's main thread at the next available opportunity, +similarly to how signals are handled. There is no interface to +determine when the code has been executed. The caller is responsible +for making sure that the file still exists whenever the remote process +tries to read it and that it hasn't been overwritten. -The remote process must be running a CPython interpreter of the same major -and minor version as the local process. If either the local or remote -interpreter is pre-release (alpha, beta, or release candidate) then the -local and remote interpreters must be the same exact version. +The remote process must be running a CPython interpreter of the same +major and minor version as the local process. If either the local or +remote interpreter is pre-release (alpha, beta, or release candidate) +then the local and remote interpreters must be the same exact version. Args: pid (int): The process ID of the target Python process. @@ -2490,7 +2464,7 @@ local and remote interpreters must be the same exact version. static PyObject * sys_remote_exec_impl(PyObject *module, int pid, PyObject *script) -/*[clinic end generated code: output=7d94c56afe4a52c0 input=39908ca2c5fe1eb0]*/ +/*[clinic end generated code: output=7d94c56afe4a52c0 input=7bd58f8da20cb74c]*/ { PyObject *path; const char *debugger_script_path; @@ -2842,8 +2816,8 @@ Sets the global lazy imports mode. The mode parameter must be one of the following strings: - "all": All top-level imports become potentially lazy -- "none": All lazy imports are suppressed (even explicitly marked ones) -- "normal": Only explicitly marked imports (with 'lazy' keyword) are lazy +- "normal": Only explicitly marked imports (with 'lazy' keyword) are + lazy In addition to the mode, lazy imports can be controlled via the filter provided to sys.set_lazy_imports_filter @@ -2852,12 +2826,12 @@ provided to sys.set_lazy_imports_filter static PyObject * sys_set_lazy_imports_impl(PyObject *module, PyObject *mode) -/*[clinic end generated code: output=1ff34ba6c4feaf73 input=f04e70d8bf9fe4f6]*/ +/*[clinic end generated code: output=1ff34ba6c4feaf73 input=db3242f0ff6e5dcc]*/ { PyImport_LazyImportsMode lazy_mode; if (!PyUnicode_Check(mode)) { PyErr_SetString(PyExc_TypeError, - "mode must be a string: 'normal', 'all', or 'none'"); + "mode must be a string: 'normal' or 'all'"); return NULL; } if (PyUnicode_CompareWithASCIIString(mode, "normal") == 0) { @@ -2866,12 +2840,9 @@ sys_set_lazy_imports_impl(PyObject *module, PyObject *mode) else if (PyUnicode_CompareWithASCIIString(mode, "all") == 0) { lazy_mode = PyImport_LAZY_ALL; } - else if (PyUnicode_CompareWithASCIIString(mode, "none") == 0) { - lazy_mode = PyImport_LAZY_NONE; - } else { PyErr_SetString(PyExc_ValueError, - "mode must be 'normal', 'all', or 'none'"); + "mode must be 'normal' or 'all'"); return NULL; } @@ -2887,22 +2858,19 @@ sys.get_lazy_imports Gets the global lazy imports mode. Returns "all" if all top level imports are potentially lazy. -Returns "none" if all explicitly marked lazy imports are suppressed. Returns "normal" if only explicitly marked imports are lazy. [clinic start generated code]*/ static PyObject * sys_get_lazy_imports_impl(PyObject *module) -/*[clinic end generated code: output=4147dec48c51ae99 input=8cb574f1e4e3003c]*/ +/*[clinic end generated code: output=4147dec48c51ae99 input=6f8dd4f2c82893f2]*/ { switch (PyImport_GetLazyImportsMode()) { case PyImport_LAZY_NORMAL: return PyUnicode_FromString("normal"); case PyImport_LAZY_ALL: return PyUnicode_FromString("all"); - case PyImport_LAZY_NONE: - return PyUnicode_FromString("none"); default: PyErr_SetString(PyExc_RuntimeError, "unknown lazy imports mode"); return NULL; @@ -2941,7 +2909,6 @@ static PyMethodDef sys_methods[] = { SYS__GETFRAME_METHODDEF SYS__GETFRAMEMODULENAME_METHODDEF SYS_GETWINDOWSVERSION_METHODDEF - SYS__ENABLELEGACYWINDOWSFSENCODING_METHODDEF SYS__IS_IMMORTAL_METHODDEF SYS_INTERN_METHODDEF SYS__IS_INTERNED_METHODDEF @@ -3436,13 +3403,6 @@ winver -- [Windows only] version number of the Python DLL\n\ " ) #endif /* MS_COREDLL */ -#ifdef MS_WINDOWS -/* concatenating string here */ -PyDoc_STR( -"_enablelegacywindowsfsencoding -- [Windows only]\n\ -" -) -#endif PyDoc_STR( "__stdin__ -- the original stdin; don't touch!\n\ __stdout__ -- the original stdout; don't touch!\n\ @@ -3531,14 +3491,39 @@ sys_set_flag(PyObject *flags, Py_ssize_t pos, PyObject *value) int _PySys_SetFlagObj(Py_ssize_t pos, PyObject *value) { - PyObject *flags = PySys_GetAttrString("flags"); - if (flags == NULL) { - return -1; + PyObject *new_flags = NULL; + PyObject *flags_str = &_Py_ID(flags); // immortal ref + + PyObject *old_flags = PySys_GetAttr(flags_str); + if (old_flags == NULL) { + goto error; } - sys_set_flag(flags, pos, value); - Py_DECREF(flags); - return 0; + new_flags = PyStructSequence_New(&FlagsType); + if (new_flags == NULL) { + goto error; + } + + for (Py_ssize_t i = 0; i < (Py_ssize_t)(Py_ARRAY_LENGTH(flags_fields) - 1); i++) { + if (i != pos) { + PyObject *old_value; + old_value = PyStructSequence_GET_ITEM(old_flags, i); // borrowed ref + sys_set_flag(new_flags, i, old_value); + } + else { + sys_set_flag(new_flags, pos, value); + } + } + + int res = _PySys_SetAttr(flags_str, new_flags); + Py_DECREF(old_flags); + Py_DECREF(new_flags); + return res; + +error: + Py_XDECREF(old_flags); + Py_XDECREF(new_flags); + return -1; } @@ -3562,8 +3547,6 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) const PyPreConfig *preconfig = &interp->runtime->preconfig; const PyConfig *config = _PyInterpreterState_GetConfig(interp); - // _PySys_UpdateConfig() modifies sys.flags in-place: - // Py_XDECREF() is needed in this case. Py_ssize_t pos = 0; #define SetFlagObj(expr) \ do { \ @@ -4074,7 +4057,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) /* implementation */ SET_SYS("implementation", make_impl_info(version_info)); - // sys.flags: updated in-place later by _PySys_UpdateConfig() + // sys.flags: updated later by _PySys_UpdateConfig() ENSURE_INFO_TYPE(FlagsType, flags_desc); SET_SYS("flags", make_flags(tstate->interp)); @@ -4194,16 +4177,21 @@ _PySys_UpdateConfig(PyThreadState *tstate) #undef COPY_LIST #undef COPY_WSTR - // sys.flags - PyObject *flags = PySys_GetAttrString("flags"); - if (flags == NULL) { + // replace sys.flags + PyObject *new_flags = PyStructSequence_New(&FlagsType); + if (new_flags == NULL) { return -1; } - if (set_flags_from_config(interp, flags) < 0) { - Py_DECREF(flags); + if (set_flags_from_config(interp, new_flags) < 0) { + Py_DECREF(new_flags); + return -1; + } + + res = _PySys_SetAttr(&_Py_ID(flags), new_flags); + Py_DECREF(new_flags); + if (res < 0) { return -1; } - Py_DECREF(flags); SET_SYS("dont_write_bytecode", PyBool_FromLong(!config->write_bytecode)); @@ -4716,7 +4704,7 @@ _PySys_SetIntMaxStrDigits(int maxdigits) // Set PyInterpreterState.long_state.max_str_digits // and PyInterpreterState.config.int_max_str_digits. PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->long_state.max_str_digits = maxdigits; - interp->config.int_max_str_digits = maxdigits; + _Py_atomic_store_int(&interp->long_state.max_str_digits, maxdigits); + _Py_atomic_store_int(&interp->config.int_max_str_digits, maxdigits); return 0; } diff --git a/Python/traceback.c b/Python/traceback.c index f0e0df7101bc21e..50a79d78d2e10ec 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1167,10 +1167,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) The caller is responsible to call PyErr_CheckSignals() to call Python signal handlers if signals were received. */ -void -_Py_DumpTraceback(int fd, PyThreadState *tstate) +const char* +PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) { dump_traceback(fd, tstate, 1); + return NULL; } #if defined(HAVE_PTHREAD_GETNAME_NP) || defined(HAVE_PTHREAD_GET_NAME_NP) @@ -1264,16 +1265,16 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) The caller is responsible to call PyErr_CheckSignals() to call Python signal handlers if signals were received. */ const char* _Py_NO_SANITIZE_THREAD -_Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, - PyThreadState *current_tstate, - Py_ssize_t max_threads) +PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, + PyThreadState *current_tstate, + Py_ssize_t max_threads) { if (max_threads == 0) { max_threads = DEFAULT_MAX_NTHREADS; } if (current_tstate == NULL) { - /* _Py_DumpTracebackThreads() is called from signal handlers by + /* PyUnstable_DumpTracebackThreads() is called from signal handlers by faulthandler. SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL are synchronous signals diff --git a/README.rst b/README.rst index 710882bc84b33d1..48f86cdb86ed1d2 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.15.0 alpha 8 +This is Python version 3.16.0 alpha 0 ===================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push @@ -136,7 +136,7 @@ What's New ---------- We have a comprehensive overview of the changes in the `What's new in Python -3.15 <https://docs.python.org/3.15/whatsnew/3.15.html>`_ document. For a more +3.16 <https://docs.python.org/3.16/whatsnew/3.16.html>`_ document. For a more detailed change log, read `Misc/NEWS <https://github.com/python/cpython/tree/main/Misc/NEWS.d>`_, but a full accounting of changes can only be gleaned from the `commit history @@ -149,7 +149,7 @@ entitled "Installing multiple versions". Documentation ------------- -`Documentation for Python 3.15 <https://docs.python.org/3.15/>`_ is online, +`Documentation for Python 3.16 <https://docs.python.org/3.16/>`_ is online, updated daily. It can also be downloaded in many formats for faster access. The documentation @@ -208,7 +208,7 @@ and ``make altinstall`` in the others. Release Schedule ---------------- -See `PEP 790 <https://peps.python.org/pep-0790/>`__ for Python 3.15 release details. +See `PEP 826 <https://peps.python.org/pep-0826/>`__ for Python 3.16 release details. Copyright and License Information diff --git a/Tools/.ruff.toml b/Tools/.ruff.toml new file mode 100644 index 000000000000000..fd9cde5e6cad650 --- /dev/null +++ b/Tools/.ruff.toml @@ -0,0 +1,14 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +[per-file-target-version] +"jit/*.py" = "py312" # Reused-quote f-strings (PEP 701) + +[lint] +select = [ + "F401", # Unused import +] + +[lint.per-file-ignores] +"c-analyzer/c_parser/__init__.py" = ["F401"] # Re-exports from submodules +"c-analyzer/c_parser/preprocessor/__init__.py" = ["F401"] # Re-exports from submodules +"c-analyzer/c_parser/preprocessor/common.py" = ["F401"] # Re-exported by preprocessor/__init__.py diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 996f725fdcb9b57..4a3dd618f6559f8 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -38,3 +38,7 @@ ignore = [ "generate_{re_casefix,sre_constants,token}.py" = [ "UP031", # Use format specifiers instead of percent format ] +"generate_slots.py" = [ + "I001", # Import block is un-sorted + "ISC003", # Explicitly concatenated string +] diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index f23c1d5286f92af..c619a9a0c1c5a1b 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -463,7 +463,7 @@ def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] - return + return # type: ignore[unreachable] try: assert spec.origin is not None st = os.stat(spec.origin) diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index f5a2f26cbb085cd..6975d9687f815dc 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -56,9 +56,6 @@ class PackageFiles(typing.NamedTuple): # values to 'exclude' if we create new files within tracked # directories that aren't sourced from third-party packages. PACKAGE_TO_FILES = { - "mpdecimal": PackageFiles( - include=["Modules/_decimal/libmpdec/**"] - ), "expat": PackageFiles( include=["Modules/expat/**"], exclude=[ diff --git a/Tools/build/generate_slots.py b/Tools/build/generate_slots.py new file mode 100755 index 000000000000000..1fdec6efa509339 --- /dev/null +++ b/Tools/build/generate_slots.py @@ -0,0 +1,399 @@ +#!/usr/bin/python +"""Generate type/module slot files +""" + +# See the input file (Python/slots.toml) for a description of its format. + +import io +import sys +import json +import tomllib +import argparse +import functools +import contextlib +import collections +from pathlib import Path + +GENERATED_BY = 'Generated by Tools/build/generate_slots.py' + +REPO_ROOT = Path(__file__).parent.parent.parent +DEFAULT_INPUT_PATH = REPO_ROOT / 'Python/slots.toml' +INCLUDE_PATH = REPO_ROOT / 'Include' +DEFAULT_PUBLIC_HEADER_PATH = INCLUDE_PATH / 'slots_generated.h' +DEFAULT_PRIVATE_HEADER_PATH = INCLUDE_PATH / 'internal/pycore_slots_generated.h' +DEFAULT_C_PATH = REPO_ROOT / 'Python/slots_generated.c' + +TABLES = { + 'tp': 'ht_type', + 'am': 'as_async', + 'nb': 'as_number', + 'mp': 'as_mapping', + 'sq': 'as_sequence', + 'bf': 'as_buffer', +} + + +class SlotInfo: + def __init__(self, id, data): + self.id = id + self.kind = data['kind'] + self._data = data + try: + self.name = data['name'] + except KeyError: + self.name = '/'.join(data["equivalents"].values()) + else: + assert self.name.isidentifier + + @functools.cached_property + def equivalents(self): + return self._data['equivalents'] + + @functools.cached_property + def dtype(self): + try: + return self._data['dtype'] + except KeyError: + if self.is_type_field: + return 'func' + raise + + @functools.cached_property + def functype(self): + return self._data['functype'] + + @functools.cached_property + def is_type_field(self): + return self._data.get('is_type_field') + + @functools.cached_property + def type_field(self): + assert self.is_type_field + return self._data.get('field', self.name.removeprefix('Py_')) + + @functools.cached_property + def type_table_ident(self): + assert self.is_type_field + return self._data.get('table', self.type_field[:2]) + + @functools.cached_property + def duplicate_handling(self): + return self._data.get('duplicates', 'reject') + + @functools.cached_property + def null_handling(self): + try: + return self._data['nulls'] + except KeyError: + if self.kind == 'compat': + return 'allow' + if self.dtype in {'ptr', 'func'}: + return 'reject' + return 'allow' + + @functools.cached_property + def must_be_static(self): + return self._data.get('must_be_static', False) + + +def parse_slots(file): + toml_contents = tomllib.load(file) + result = [None] * len(toml_contents) + for key, data in toml_contents.items(): + slot_id = int(key) + try: + if result[slot_id]: + raise ValueError(f'slot ID {slot_id} repeated') + result[slot_id] = SlotInfo(slot_id, data) + except Exception as e: + e.add_note(f'handling slot {slot_id}') + raise + return result + + +class CWriter: + """Simple helper for generating C code""" + + def __init__(self, file): + self.file = file + self.indent = '' + self(f'/* {GENERATED_BY} */') + self() + + def out(self, *args, **kwargs): + """print args to the file, with current indent at the start""" + print(self.indent, end='', file=self.file) + print(*args, file=self.file, **kwargs) + + __call__ = out + + @contextlib.contextmanager + def block(self, header=None, end=''): + """Context for a {}-enclosed block of C""" + if header is None: + self.out('{') + else: + self.out(header, '{') + old_indent = self.indent + self.indent += ' ' + yield + self.indent = old_indent + self.out('}' + end) + + +def write_public_header(f, slots): + out = CWriter(f) + out(f'#ifndef _PY_HAVE_SLOTS_GENERATED_H') + out(f'#define _PY_HAVE_SLOTS_GENERATED_H') + out() + out(f'#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15)') + out(f'#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) NEW') + out(f'#else') + out(f'#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) OLD') + out(f'#endif') + out() + compat_ids = {} + for slot in slots: + if slot.kind == 'compat': + for new_name in slot.equivalents.values(): + compat_ids[new_name] = slot.id + for slot in slots: + if slot.kind == 'compat': + continue + slot_id = slot.id + if compat := compat_ids.get(slot.name): + slot_id = f'_Py_SLOT_COMPAT_VALUE({compat}, {slot_id})' + out(f'#define {slot.name} {slot_id}') + out() + out(f'#define _Py_slot_COUNT {len(slots)}') + out(f'#endif /* _PY_HAVE_SLOTS_GENERATED_H */') + + +def write_private_header(f, slots): + out = CWriter(f) + + def add_case(slot): + out(out(f' case {slot.id}:')) + + slots_by_name = {slot.name: slot for slot in slots} + + out(f'#ifndef _PY_HAVE_INTERNAL_SLOTS_GENERATED_H') + out(f'#define _PY_HAVE_INTERNAL_SLOTS_GENERATED_H') + for kind in 'type', 'mod': + out() + out(f'static inline uint16_t') + out(f'_PySlot_resolve_{kind}_slot(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + good_slots = [] + for slot in slots: + if slot.kind == 'compat': + new_slot = slots_by_name[slot.equivalents[kind]] + out(f'case {slot.id}:') + out(f' return {new_slot.name};') + elif slot.kind in {kind, 'slot'}: + good_slots.append(f'case {slot.name}:') + for case in good_slots: + out(case) + out(f' return slot_id;') + out(f'default:') + out(f' return Py_slot_invalid;') + out() + out(f'static inline void*') + out(f'_PySlot_type_getslot(PyTypeObject *tp, uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + for slot in slots: + if slot.is_type_field: + field = slot.type_field + table_ident = slot.type_table_ident + if table_ident == 'tp': + out(f'case {slot.name}:') + out(f' return (void*)tp->{field};') + else: + if table_ident == 'ht': + cond = 'tp->tp_flags & Py_TPFLAGS_HEAPTYPE' + val = f'((PyHeapTypeObject*)tp)->{field}' + else: + table = TABLES[table_ident] + cond = f'tp->tp_{table}' + val = f'tp->tp_{table}->{field}' + out(f'case {slot.name}:') + out(f' if (!({cond})) return NULL;') + out(f' return (void*){val};') + out(f'_PySlot_err_bad_slot("PyType_GetSlot", slot_id);') + out(f'return NULL;') + out() + out(f'static inline void') + out(f'_PySlot_heaptype_apply_field_slot(PyHeapTypeObject *ht,', + f'PySlot slot)') + with out.block(): + with out.block('switch (slot.sl_id)'): + for slot in slots: + if slot.is_type_field: + field = slot.type_field + table_ident = slot.type_table_ident + if table_ident == 'ht': + continue + table = TABLES[table_ident] + if slot.dtype == 'func': + functype = f'({slot.functype})' + else: + functype = '' + out(f'case {slot.name}:') + out(f' ht->{table}.{field} = {functype}slot.sl_{slot.dtype};') + out(f' break;') + out() + out(f'static inline _PySlot_DTYPE') + out(f'_PySlot_get_dtype(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + for slot in slots: + if slot.kind == 'compat': + continue + dtype = slot.dtype + name = slot.name + out(f'case {name}: return _PySlot_DTYPE_{dtype.upper()};') + out(f'default: return _PySlot_DTYPE_VOID;') + out() + out(f'static inline _PySlot_PROBLEM_HANDLING') + out(f'_PySlot_get_duplicate_handling(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + results = collections.defaultdict(list) + for slot in slots: + if slot.kind == 'compat': + continue + handling = slot.duplicate_handling + results[handling.upper()].append(f'case {slot.name}:') + results.pop('REJECT') + for handling, cases in results.items(): + for case in cases: + out(case) + out(f' return _PySlot_PROBLEM_{handling};') + out(f'default:') + out(f' return _PySlot_PROBLEM_REJECT;') + out() + out(f'static inline _PySlot_PROBLEM_HANDLING') + out(f'_PySlot_get_null_handling(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + results = collections.defaultdict(list) + for slot in slots: + if slot.kind == 'compat': + continue + handling = slot.null_handling + if handling is None: + if slot.kind != 'compat' and slot.dtype in {'ptr', 'func'}: + handling = 'reject' + else: + handling = 'allow' + results[handling.upper()].append(f'case {slot.name}:') + results.pop('REJECT') + for handling, cases in results.items(): + for case in cases: + out(case) + out(f' return _PySlot_PROBLEM_{handling};') + out(f'default:') + out(f' return _PySlot_PROBLEM_REJECT;') + out() + out(f'static inline bool') + out(f'_PySlot_get_must_be_static(uint16_t slot_id)') + with out.block(): + with out.block('switch (slot_id)'): + cases = [] + for slot in slots: + if slot.must_be_static: + out(f'case {slot.name}: return true;') + out(f'return false;') + out() + out(f'#endif /* _PY_HAVE_INTERNAL_SLOTS_GENERATED_H */') + + +def write_c(f, slots): + out = CWriter(f) + out('#include "Python.h"') + out('#include "pycore_slots.h" // _PySlot_names') + out() + with out.block(f'const char *const _PySlot_names[] =', end=';'): + for slot in slots: + out(f'"{slot.name}",') + out('NULL') + + +@contextlib.contextmanager +def replace_file(filename): + file_path = Path(filename) + with io.StringIO() as sio: + yield sio + try: + old_text = file_path.read_text() + except FileNotFoundError: + old_text = None + new_text = sio.getvalue() + if old_text == new_text: + print(f'{filename}: not modified', file=sys.stderr) + else: + print(f'{filename}: writing new content', file=sys.stderr) + file_path.write_text(new_text) + + +def main(argv): + if len(argv) == 1: + # No sens calling this with no arguments. + argv.append('--help') + + parser = argparse.ArgumentParser(prog=argv[0], description=__doc__) + parser.add_argument( + '-i', '--input', default=DEFAULT_INPUT_PATH, + help=f'the input file (default: {DEFAULT_INPUT_PATH})') + parser.add_argument( + '--generate-all', action=argparse.BooleanOptionalAction, + help='write all output files to their default locations') + parser.add_argument( + '-j', '--jsonl', action=argparse.BooleanOptionalAction, + help='write info to stdout in "JSON Lines" format (one JSON per line)') + outfile_group = parser.add_argument_group( + 'output files', + description='By default, no files are generated. Use --generate-all ' + + 'or the options below to generate them.') + outfile_group.add_argument( + '-H', '--public-header', + help='file into which to write the public header') + outfile_group.add_argument( + '-I', '--private-header', + help='file into which to write the private header') + outfile_group.add_argument( + '-C', '--cfile', + help='file into which to write internal C code') + args = parser.parse_args(argv[1:]) + + if args.generate_all: + if args.public_header is None: + args.public_header = DEFAULT_PUBLIC_HEADER_PATH + if args.private_header is None: + args.private_header = DEFAULT_PRIVATE_HEADER_PATH + if args.cfile is None: + args.cfile = DEFAULT_C_PATH + + with open(args.input, 'rb') as f: + slots = parse_slots(f) + + if args.jsonl: + for slot in slots: + print(json.dumps(slot.to_dict())) + + if args.public_header: + with replace_file(args.public_header) as f: + write_public_header(f, slots) + + if args.private_header: + with replace_file(args.private_header) as f: + write_private_header(f, slots) + + if args.cfile: + with replace_file(args.cfile) as f: + write_c(f, slots) + +if __name__ == "__main__": + main(sys.argv) diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index bda725396406118..f8828a56b4c7da7 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -42,6 +42,7 @@ 'test', 'xxlimited', 'xxlimited_35', + 'xxlimited_3_13', 'xxsubtype', } diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 5465e2d4b6171f1..485c9314cf70015 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -24,8 +24,6 @@ python_version = 3.10 # ...And be strict: strict = True -strict_bytes = True -local_partial_types = True extra_checks = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True diff --git a/Tools/c-analyzer/c_parser/preprocessor/clang.py b/Tools/c-analyzer/c_parser/preprocessor/clang.py index 574a23f8f6d6f9f..738c261d1ecd80a 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/clang.py +++ b/Tools/c-analyzer/c_parser/preprocessor/clang.py @@ -1,5 +1,4 @@ import os.path -import re, sys from . import common as _common from . import gcc as _gcc diff --git a/Tools/c-analyzer/c_parser/preprocessor/gcc.py b/Tools/c-analyzer/c_parser/preprocessor/gcc.py index 4a55a1a24ee1bed..92134bc1321e1b5 100644 --- a/Tools/c-analyzer/c_parser/preprocessor/gcc.py +++ b/Tools/c-analyzer/c_parser/preprocessor/gcc.py @@ -11,6 +11,7 @@ '_testclinic_limited.c', 'xxlimited.c', 'xxlimited_35.c', + 'xxlimited_3_13.c', )) # C files in the fhe following directories must not be built with diff --git a/Tools/c-analyzer/cpython/_analyzer.py b/Tools/c-analyzer/cpython/_analyzer.py index 43ed552fcf75d90..404a81af11e39ff 100644 --- a/Tools/c-analyzer/cpython/_analyzer.py +++ b/Tools/c-analyzer/cpython/_analyzer.py @@ -77,6 +77,7 @@ 'PyStructSequence_Field[]', 'PyStructSequence_Desc', 'PyABIInfo', + 'PySlot[]', } # XXX We should normalize all cases to a single name, diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index fc3cbf3779db26d..2875f45cb8d3756 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -118,8 +118,6 @@ def format_tsv_lines(lines): ('*', './Include/internal'), ('*', './Include/internal/mimalloc'), - ('Modules/_decimal/**/*.c', 'Modules/_decimal/libmpdec'), - ('Modules/_elementtree.c', 'Modules/expat'), ('Modules/pyexpat.c', 'Modules/expat'), @@ -279,8 +277,6 @@ def format_tsv_lines(lines): # ('Modules/_dbmmodule.c', 'USE_BERKDB', '1'), # See: setup.py - ('Modules/_decimal/**/*.c', 'CONFIG_64', '1'), - ('Modules/_decimal/**/*.c', 'ASM', '1'), ('Modules/expat/xmlparse.c', 'HAVE_EXPAT_CONFIG_H', '1'), ('Modules/expat/xmlparse.c', 'XML_POOR_ENTROPY', '1'), ('Modules/_dbmmodule.c', 'HAVE_GDBM_DASH_NDBM_H', '1'), @@ -314,6 +310,7 @@ def format_tsv_lines(lines): # default: (10_000, 200) # First match wins. _abs('Modules/_ctypes/ctypes.h'): (5_000, 500), + _abs('Modules/_cursesmodule.c'): (20_000, 300), _abs('Modules/_datetimemodule.c'): (20_000, 300), _abs('Modules/_hacl/*.c'): (200_000, 500), _abs('Modules/posixmodule.c'): (20_000, 500), diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index aa89e312b624825..bf08e5568205e7a 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -327,6 +327,7 @@ Modules/pyexpat.c - error_info_of - Modules/pyexpat.c - handler_info - Modules/termios.c - termios_constants - Modules/timemodule.c init_timezone YEAR - +Modules/unicodedata.c unicodedata_create_capi capi - Objects/bytearrayobject.c - _PyByteArray_empty_string - Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - @@ -462,10 +463,12 @@ Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - Modules/_testcapi/heaptype.c - _testcapimodule - Modules/_testcapi/mem.c - FmData - Modules/_testcapi/mem.c - FmHook - +Modules/_testcapi/module.c module_from_def_nonstatic_nested subslots - Modules/_testcapi/object.c - MyObject_dealloc_called - Modules/_testcapi/object.c - MyType - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watch_once - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed - @@ -576,6 +579,7 @@ Modules/_testinternalcapi.c - Test_EvalFrame_Resumes - Modules/_testinternalcapi.c - Test_EvalFrame_Loads - Modules/_testinternalcapi/interpreter.c - Test_EvalFrame_Resumes - Modules/_testinternalcapi/interpreter.c - Test_EvalFrame_Loads - +Modules/_testlimitedcapi/slots.c - TestMethods - Modules/_testmultiphase.c - Example_Type_slots - Modules/_testmultiphase.c - Example_Type_spec - Modules/_testmultiphase.c - Example_methods - @@ -618,6 +622,13 @@ Modules/_testmultiphase.c - slots_exec_unreported_exception - Modules/_testmultiphase.c - slots_nonmodule_with_exec_slots - Modules/_testmultiphase.c - testexport_methods - Modules/_testmultiphase.c - uninitialized_def - +Modules/_testmultiphase.c PyModExport__test_from_modexport slots - +Modules/_testmultiphase.c PyModExport__test_from_modexport_gil_used slots - +Modules/_testmultiphase.c PyModExport__test_from_modexport_create_nonmodule slots - +Modules/_testmultiphase.c PyModExport__test_from_modexport_create_nonmodule_gil_used slots - +Modules/_testmultiphase.c PyModExport__test_from_modexport_smoke slots - +Modules/_testmultiphase.c - modexport_empty_slots - +Modules/_testmultiphase.c - modexport_minimal_slots - Modules/_testsinglephase.c - global_state - Modules/_testsinglephase.c - static_module_circular - Modules/_xxtestfuzz/_xxtestfuzz.c - _fuzzmodule - @@ -776,3 +787,4 @@ Objects/dictobject.c - PyFrozenDict_Type - ## False positives Python/specialize.c - _Py_InitCleanup - +Python/pystate.c - _no_tstate_sentinel - diff --git a/Tools/c-analyzer/distutils/util.py b/Tools/c-analyzer/distutils/util.py index f687a28ec2f40e8..c8e92658d953661 100644 --- a/Tools/c-analyzer/distutils/util.py +++ b/Tools/c-analyzer/distutils/util.py @@ -8,7 +8,6 @@ import re import string import sys -from distutils.errors import DistutilsPlatformError def get_host_platform(): """Return a string that identifies the current platform. This is used mainly to diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 414ca18be4654cf..4eed4d8d60e4856 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -322,6 +322,16 @@ class Family: size: str members: list[Instruction] + def get_member_record_names(self) -> tuple[str, ...]: + seen: set[str] = set() + names: list[str] = [] + for member in self.members: + for part in member.parts: + if part.properties.records_value and part.name not in seen: + seen.add(part.name) + names.append(part.name) + return tuple(names) + def dump(self, indent: str) -> None: print(indent, self.name, "= ", ", ".join([m.name for m in self.members])) @@ -599,6 +609,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_CLEAR", "PyStackRef_CLOSE_SPECIALIZED", "PyStackRef_DUP", + "PyStackRef_DupImmortal", "PyStackRef_False", "PyStackRef_FromPyObjectBorrow", "PyStackRef_FromPyObjectNew", @@ -616,6 +627,9 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_RefcountOnObject", "PyStackRef_TYPE", "PyStackRef_True", + "PyBytes_GET_SIZE", + "PyDict_GET_SIZE", + "PySet_GET_SIZE", "PyTuple_GET_ITEM", "PyTuple_GET_SIZE", "PyType_HasFeature", @@ -643,6 +657,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyFrame_PushUnchecked", "_PyFrame_SetStackPointer", "_PyFrame_StackPush", + "_PyFrame_StackAssertInvalid", "_PyFunction_SetVersion", "_PyGen_GetGeneratorFromFrame", "gen_try_set_executing", @@ -717,6 +732,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_Py_GatherStats_GetIter", "_PyStolenTuple_Free", "PyObject_GC_UnTrack", + "_PyErr_ExceptionMatches", ) @@ -991,7 +1007,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: eval_breaker="CHECK_PERIODIC" in op.name, needs_this=variable_used(op, "this_instr"), always_exits=always_exits(op), - sync_sp=variable_used(op, "SYNC_SP"), + sync_sp=variable_used(op, "SYNC_SP") or variable_used(op, "SAVE_STACK") or variable_used(op, "RELOAD_STACK"), uses_co_consts=variable_used(op, "FRAME_CO_CONSTS"), uses_co_names=variable_used(op, "FRAME_CO_NAMES"), uses_locals=variable_used(op, "GETLOCAL") and not has_free, diff --git a/Tools/cases_generator/cwriter.py b/Tools/cases_generator/cwriter.py index 6636755db55eecb..de6247185824497 100644 --- a/Tools/cases_generator/cwriter.py +++ b/Tools/cases_generator/cwriter.py @@ -39,7 +39,6 @@ def set_position(self, tkn: Token) -> None: self.newline = False def emit_at(self, txt: str, where: Token) -> None: - self.maybe_write_spill() self.set_position(where) self.out.write(txt) @@ -116,7 +115,6 @@ def emit_str(self, txt: str) -> None: self.last_token = None def emit(self, txt: str | Token) -> None: - self.maybe_write_spill() if isinstance(txt, Token): self.emit_token(txt) elif isinstance(txt, str): @@ -130,28 +128,6 @@ def start_line(self) -> None: self.newline = True self.last_token = None - def emit_spill(self) -> None: - if self.pending_reload: - self.pending_reload = False - return - assert not self.pending_spill - self.pending_spill = True - - def maybe_write_spill(self) -> None: - if self.pending_spill: - self.pending_spill = False - self.emit_str("_PyFrame_SetStackPointer(frame, stack_pointer);\n") - elif self.pending_reload: - self.pending_reload = False - self.emit_str("stack_pointer = _PyFrame_GetStackPointer(frame);\n") - - def emit_reload(self) -> None: - if self.pending_spill: - self.pending_spill = False - return - assert not self.pending_reload - self.pending_reload = True - @contextlib.contextmanager def header_guard(self, name: str) -> Iterator[None]: self.out.write( diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index bdc4324b6702006..ccff95a961ce1ea 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -7,14 +7,12 @@ analysis_error, Label, CodeSection, - Uop, ) from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable from lexer import Token from stack import Storage, StackError from parser import Stmt, SimpleStmt, BlockStmt, IfStmt, ForStmt, WhileStmt, MacroIfStmt -from stack import PRINT_STACKS DEBUG = False class TokenIterator: @@ -505,7 +503,8 @@ def reload_stack( next(tkn_iter) next(tkn_iter) next(tkn_iter) - self.emit_reload(storage) + storage.flush(self.out) + storage.full_reload(self.out) return True def instruction_size(self, @@ -711,7 +710,11 @@ def emit_ForStmt( self.out.emit(stmt.for_) for tkn in stmt.header: self.out.emit(tkn) - return self._emit_stmt(stmt.body, uop, storage, inst) + reachable, brace, body_storage = self._emit_stmt(stmt.body, uop, storage.copy(), inst) + body_storage.merge(storage, self.out) + if brace is not None: + self.out.emit(brace) + return reachable, None, storage def emit_WhileStmt( self, diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 21407ad7df1e9a5..00f6804f1724b59 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -19,7 +19,6 @@ cflags, ) from cwriter import CWriter -from dataclasses import dataclass from typing import TextIO from stack import get_stack_effect diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index aa914783f7cdc2d..a32e6457cded265 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -112,6 +112,7 @@ def decref_inputs( def emit_default(out: CWriter, uop: Uop, stack: Stack) -> None: + assert stack.physical_sp is not None null = CWriter.null() for var in reversed(uop.stack.inputs): stack.pop(var, null) @@ -153,6 +154,19 @@ def emit_save(self, storage: Storage) -> None: def emit_reload(self, storage: Storage) -> None: pass + def reload_stack( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + return True + def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: self.out.emit(goto) self.out.emit(label) diff --git a/Tools/cases_generator/parser.py b/Tools/cases_generator/parser.py index ccf8bf649520ff6..aa6c0b1446fb76b 100644 --- a/Tools/cases_generator/parser.py +++ b/Tools/cases_generator/parser.py @@ -20,7 +20,6 @@ MacroIfStmt, ) -import pprint CodeDef = InstDef | LabelDef diff --git a/Tools/cases_generator/record_function_generator.py b/Tools/cases_generator/record_function_generator.py index 6f518ffdcf2ac20..b5a028384adb491 100644 --- a/Tools/cases_generator/record_function_generator.py +++ b/Tools/cases_generator/record_function_generator.py @@ -3,7 +3,9 @@ from analyzer import ( Analysis, + Family, Instruction, + Uop, analyze_files, CodeSection, ) @@ -19,7 +21,6 @@ from cwriter import CWriter from tier1_generator import write_uop, Emitter, declare_variable -from typing import TextIO from lexer import Token from stack import Stack, Storage @@ -28,16 +29,6 @@ # Must match MAX_RECORDED_VALUES in Include/internal/pycore_optimizer.h. MAX_RECORDED_VALUES = 3 -# Map `_RECORD_*` uops to the helper that converts a raw family-recorded -# value to the form the specialized member consumes. -_RECORD_TRANSFORM_HELPERS: dict[str, str] = { - "_RECORD_TOS_TYPE": "record_trace_transform_to_type", - "_RECORD_NOS_TYPE": "record_trace_transform_to_type", - "_RECORD_NOS_GEN_FUNC": "record_trace_transform_gen_func", - "_RECORD_3OS_GEN_FUNC": "record_trace_transform_gen_func", - "_RECORD_BOUND_METHOD": "record_trace_transform_bound_method", -} - # Recorder uops whose slot kind differs from the leading word of their name. _RECORD_SLOT_KIND_OVERRIDES: dict[str, str] = { "_RECORD_BOUND_METHOD": "CALLABLE", @@ -45,9 +36,11 @@ class RecorderEmitter(Emitter): - def __init__(self, out: CWriter): + def __init__(self, out: CWriter, target: str, incref: str): super().__init__(out, {}) self._replacers["RECORD_VALUE"] = self.record_value + self.target = target + self.incref = incref def record_value( self, @@ -57,13 +50,13 @@ def record_value( storage: Storage, inst: Instruction | None, ) -> bool: - lparen = next(tkn_iter) + next(tkn_iter) self.out.start_line() - self.emit("*recorded_value = (PyObject *)") + self.emit(f"{self.target} = (PyObject *)") emit_to(self.out, tkn_iter, "RPAREN") next(tkn_iter) # Semi colon self.emit(";\n") - self.emit("Py_INCREF(*recorded_value);\n") + self.emit(f"{self.incref}({self.target});\n") return True @@ -80,39 +73,40 @@ def get_instruction_record_names(inst: Instruction) -> list[str]: def get_family_record_names( - family_head: Instruction, - family_members: list[Instruction], + family: Family, instruction_records: dict[str, list[str]], record_slot_keys: dict[str, str], ) -> list[str]: - member_records = [instruction_records[m.name] for m in family_members] - all_member_names = {n for names in member_records for n in names} + family_record_names = set(family.get_member_record_names()) + family_record_names.update(instruction_records[family.name]) records: list[str] = [] slot_index: dict[str, int] = {} def add(name: str) -> None: kind = record_slot_keys[name] - # Prefer the raw recorder if any member uses it; otherwise the given form. + # Prefer the raw recorder if any family instruction uses it. raw = f"_RECORD_{kind}" - source = raw if raw in all_member_names else name + source = raw if raw in family_record_names else name existing = slot_index.get(kind) if existing is None: slot_index[kind] = len(records) records.append(source) elif records[existing] != source: - raise ValueError( - f"Family {family_head.name} has incompatible recorders for " - f"slot {kind}: {records[existing]} and {source}" - ) - - for names in member_records: - for name in names: + if raw not in record_slot_keys: + raise ValueError( + f"Family {family.name} has incompatible recorders for " + f"slot {kind}: {records[existing]} and {source}, " + f"and no raw recorder {raw} exists to use as a base." + ) + records[existing] = raw + + for member in family.members: + for name in instruction_records[member.name]: add(name) - # Family head supplies any slots no member exercises. - for name in instruction_records[family_head.name]: - if record_slot_keys[name] not in slot_index: - slot_index[record_slot_keys[name]] = len(records) - records.append(name) + # Family head supplies any slots no member exercises, and may also + # conflict with members (resolved via the raw recorder above). + for name in instruction_records[family.name]: + add(name) return records @@ -121,10 +115,11 @@ def get_record_consumer_layout( source_records: list[str], own_records: list[str], record_slot_keys: dict[str, str], -) -> tuple[list[int], int]: +) -> tuple[list[int], int, list[str]]: used = [False] * len(source_records) slot_map: list[int] = [] transform_mask = 0 + transform_names: list[str] = [] for i, own in enumerate(own_records): own_kind = record_slot_keys[own] for j, src in enumerate(source_records): @@ -133,13 +128,43 @@ def get_record_consumer_layout( slot_map.append(j) if src != own: transform_mask |= 1 << i + if own not in transform_names: + transform_names.append(own) break else: raise ValueError( f"Instruction {inst_name} has no compatible family slot for " f"{own} in {source_records}" ) - return slot_map, transform_mask + return slot_map, transform_mask, transform_names + + +def get_record_transform_input(uop: Uop) -> str: + inputs = [var for var in uop.stack.inputs if var.used] + if len(inputs) != 1 or inputs[0].is_array(): + raise ValueError( + f"Recorder transform for {uop.name} needs exactly one scalar input" + ) + return inputs[0].name + + +def generate_record_transform_function(uop: Uop, out: CWriter) -> None: + input_name = get_record_transform_input(uop) + out.emit("static PyObject *\n") + out.emit(f"_PyOpcode_RecordTransform{uop.name[7:]}(PyObject *recorded_value)\n") + out.emit("{\n") + out.emit("PyObject *transformed_value = NULL;\n") + for var in uop.stack.inputs: + if var.used: + declare_variable(var, out) + out.emit(f"{input_name} = PyStackRef_FromPyObjectBorrow(recorded_value);\n") + emitter = RecorderEmitter(out, "transformed_value", "Py_XINCREF") + emitter.emit_tokens(uop, Storage(Stack(), [], [], 0, False), None, False) + out.start_line() + out.emit("Py_DECREF(recorded_value);\n") + out.emit("return transformed_value;\n") + out.emit("}\n\n") + def generate_recorder_functions(filenames: list[str], analysis: Analysis, out: CWriter) -> None: write_header(__file__, filenames, out.out) @@ -151,7 +176,7 @@ def generate_recorder_functions(filenames: list[str], analysis: Analysis, out: C """ ) args = "_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int oparg, PyObject **recorded_value" - emitter = RecorderEmitter(out) + emitter = RecorderEmitter(out, "*recorded_value", "Py_INCREF") nop = analysis.instructions["NOP"] for uop in analysis.uops.values(): if not uop.properties.records_value: @@ -179,8 +204,7 @@ def generate_recorder_tables(analysis: Analysis, out: CWriter) -> None: record_slot_keys = {name: get_record_slot_kind(name) for name in record_uop_names} family_record_table = { family.name: get_family_record_names( - analysis.instructions[family.name], - family.members, + family, instruction_records, record_slot_keys, ) @@ -190,12 +214,12 @@ def generate_recorder_tables(analysis: Analysis, out: CWriter) -> None: record_table: dict[str, list[str]] = {} record_consumer_table: dict[str, tuple[list[int], int]] = {} record_function_indexes: dict[str, int] = {} + record_transform_names: list[str] = [] for inst in analysis.instructions.values(): own_records = instruction_records[inst.name] # TRACE_RECORD runs before execution, but specialization may rewrite - # the opcode before translation. Record the raw family shape (union - # of head + members) so any opcode in the family can be translated - # from the same recorded layout. + # the opcode before translation. Use the shared family recording shape + # so any opcode in the family can be translated from the same layout. family = inst.family or analysis.families.get(inst.name) records = family_record_table[family.name] if family is not None else own_records if not records: @@ -210,9 +234,13 @@ def generate_recorder_tables(analysis: Analysis, out: CWriter) -> None: if name not in record_function_indexes: record_function_indexes[name] = len(record_function_indexes) + 1 if own_records: - record_consumer_table[inst.name] = get_record_consumer_layout( + slots, mask, transform_names = get_record_consumer_layout( inst.name, records, own_records, record_slot_keys ) + record_consumer_table[inst.name] = (slots, mask) + for name in transform_names: + if name not in record_transform_names: + record_transform_names.append(name) for name, index in record_function_indexes.items(): out.emit(f"#define {name}_INDEX {index}\n") @@ -240,37 +268,32 @@ def generate_recorder_tables(analysis: Analysis, out: CWriter) -> None: for name in record_function_indexes: out.emit(f" [{name}_INDEX] = _PyOpcode_RecordFunction{name[7:]},\n") out.emit("};\n") - generate_record_transform_dispatcher(record_uop_names, out) + out.emit("\n") + for name in record_transform_names: + generate_record_transform_function(analysis.uops[name], out) + generate_record_transform_dispatcher(record_transform_names, out) def generate_record_transform_dispatcher( - record_uop_names: list[str], out: CWriter + transform_names: list[str], out: CWriter ) -> None: """Emit a switch that converts a family-recorded value for a recorder uop. - Only `_RECORD_*` uops that need conversion get a case; the default - returns the input value unchanged. Helpers live in Python/optimizer.c. + Only `_RECORD_*` uops that need conversion get a case; the default returns + the input value unchanged. """ - cases: dict[str, list[str]] = {} - for record_name in record_uop_names: - helper = _RECORD_TRANSFORM_HELPERS.get(record_name) - if helper is None: - continue - cases.setdefault(helper, []).append(record_name) - out.emit("\n") out.emit( "PyObject *\n" "_PyOpcode_RecordTransformValue(int uop, PyObject *value)\n" "{\n" ) - out.emit(" switch (uop) {\n") - for helper, names in cases.items(): - for name in names: - out.emit(f" case {name}:\n") - out.emit(f" return {helper}(value);\n") - out.emit(" default:\n") - out.emit(" return value;\n") - out.emit(" }\n") + out.emit("switch (uop) {\n") + for name in transform_names: + out.emit(f"case {name}:\n") + out.emit(f" return _PyOpcode_RecordTransform{name[7:]}(value);\n") + out.emit("default:\n") + out.emit(" return value;\n") + out.emit("}\n") out.emit("}\n") diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py index efc534fb607b902..8044ac10dd0ad70 100644 --- a/Tools/cases_generator/stack.py +++ b/Tools/cases_generator/stack.py @@ -218,7 +218,8 @@ def array_or_scalar(var: StackItem | Local) -> str: class Stack: def __init__(self, check_stack_bounds: bool = False) -> None: self.base_offset = PointerOffset.zero() - self.physical_sp = PointerOffset.zero() + self.physical_sp:PointerOffset | None = PointerOffset.zero() + self.frame_sp:PointerOffset | None = None self.logical_sp = PointerOffset.zero() self.variables: list[Local] = [] self.check_stack_bounds = check_stack_bounds @@ -239,6 +240,7 @@ def drop(self, var: StackItem, check_liveness: bool) -> None: raise StackError(f"Dropping live value '{var.name}'") def pop(self, var: StackItem, out: CWriter) -> Local: + assert self.physical_sp is not None if self.variables: top = self.variables[-1] if var.is_array() != top.is_array() or top.size != var.size: @@ -299,6 +301,8 @@ def _do_emit( out.emit(f"stack_pointer[{stack_offset.to_c()}] = {var.name};\n") def _save_physical_sp(self, out: CWriter) -> None: + if self.physical_sp is None: + return if self.physical_sp != self.logical_sp: diff = self.logical_sp - self.physical_sp out.start_line() @@ -311,6 +315,7 @@ def save_variables(self, out: CWriter) -> None: out.start_line() var_offset = self.base_offset for var in self.variables: + assert self.physical_sp is not None if ( var.in_local and not var.memory_offset and @@ -324,6 +329,8 @@ def save_variables(self, out: CWriter) -> None: var_offset = var_offset.push(var.item) def stack_bound_check(self, out: CWriter) -> None: + if self.physical_sp is None: + return if not self.check_stack_bounds: return if self.physical_sp != self.logical_sp: @@ -338,6 +345,44 @@ def flush(self, out: CWriter) -> None: self._save_physical_sp(out) out.start_line() + def emit_spill(self, out: CWriter) -> None: + if self.frame_sp != self.physical_sp: + out.emit_str("_PyFrame_SetStackPointer(frame, stack_pointer);\n") + self.frame_sp = self.physical_sp + else: + out.emit_str("assert(stack_pointer == _PyFrame_GetStackPointer(frame));\n") + out.emit_str("_PyFrame_StackPointerValidate(frame);\n"); + + def expected_frame_pointer(self) -> str: + assert self.frame_sp is not None + assert self.physical_sp is not None + diff = self.frame_sp - self.physical_sp + return f"(stack_pointer + {diff.to_c()})" + + def emit_reload(self, out: CWriter) -> None: + if self.physical_sp is None: + out.emit_str("stack_pointer = _PyFrame_GetStackPointer(frame);\n") + if self.frame_sp is None: + self.frame_sp = PointerOffset.zero() + self.physical_sp = self.frame_sp + out.emit_str("_PyFrame_StackPointerInvalidate(frame);\n"); + + def emit_fullreload(self, out: CWriter, spilled: bool = True) -> None: + self._print(out) + while self.variables: + top = self.variables.pop() + if not top.in_memory(): + raise StackError(f"Cannot reload with live variable: {top.name}") + self.base_offset = PointerOffset.zero() + self.physical_sp = PointerOffset.zero() + self.frame_sp = PointerOffset.zero() + self.logical_sp = PointerOffset.zero() + out.emit_str("stack_pointer = _PyFrame_GetStackPointer(frame);\n") + if spilled: + out.emit_str("_PyFrame_StackPointerInvalidate(frame);\n") + else: + out.emit_str("_PyFrame_StackAssertInvalid(frame);\n") + def is_flushed(self) -> bool: for var in self.variables: if not var.in_memory(): @@ -345,12 +390,15 @@ def is_flushed(self) -> bool: return self.physical_sp == self.logical_sp def sp_offset(self) -> str: + assert self.physical_sp is not None return (self.physical_sp - self.logical_sp).to_c() def as_comment(self) -> str: variables = ", ".join([v.compact_str() for v in self.variables]) + sp = "undefined" if self.physical_sp is None else self.physical_sp.to_c() + fp = "undefined" if self.frame_sp is None else self.frame_sp.to_c() return ( - f"/* Variables=[{variables}]; base={self.base_offset.to_c()}; sp={self.physical_sp.to_c()}; logical_sp={self.logical_sp.to_c()} */" + f"/* Variables=[{variables}]; base={self.base_offset.to_c()}; sp={sp}; logical_sp={self.logical_sp.to_c()}; frame_sp={fp} */" ) def _print(self, out: CWriter) -> None: @@ -362,6 +410,7 @@ def copy(self) -> "Stack": other.base_offset = self.base_offset other.physical_sp = self.physical_sp other.logical_sp = self.logical_sp + other.frame_sp = self.frame_sp other.variables = [var.copy() for var in self.variables] other.check_stack_bounds = self.check_stack_bounds return other @@ -379,12 +428,18 @@ def __eq__(self, other: object) -> bool: def align(self, other: "Stack", out: CWriter) -> None: if self.logical_sp != other.logical_sp: raise StackError("Cannot align stacks: differing logical top") + if self.physical_sp is None and other.physical_sp is None: + return + if self.physical_sp is None or other.physical_sp is None: + raise StackError("Cannot align stacks: stack_pointer is partly undefined") if self.physical_sp == other.physical_sp: return diff = other.physical_sp - self.physical_sp out.start_line() out.emit(f"stack_pointer += {diff.to_c()};\n") self.physical_sp = other.physical_sp + if self.frame_sp != other.frame_sp: + self.frame_sp = None def merge(self, other: "Stack", out: CWriter) -> None: if len(self.variables) != len(other.variables): @@ -402,6 +457,9 @@ def merge(self, other: "Stack", out: CWriter) -> None: raise StackError(f"Mismatched stack depths for {self_var.name}: {self_var.memory_offset} and {other_var.memory_offset}") elif other_var.memory_offset is None: self_var.memory_offset = None + if self.frame_sp != other.frame_sp: + self.frame_sp = None + other.frame_sp = None def stacks(inst: Instruction | PseudoInstruction) -> Iterator[StackEffect]: @@ -525,7 +583,7 @@ def save(self, out: CWriter) -> None: assert self.spilled >= 0 if self.spilled == 0: out.start_line() - out.emit_spill() + self.stack.emit_spill(out) self.spilled += 1 def save_inputs(self, out: CWriter) -> None: @@ -534,7 +592,7 @@ def save_inputs(self, out: CWriter) -> None: self.clear_dead_inputs() self.stack.flush(out) out.start_line() - out.emit_spill() + self.stack.emit_spill(out) self.spilled += 1 def reload(self, out: CWriter) -> None: @@ -544,10 +602,25 @@ def reload(self, out: CWriter) -> None: self.spilled -= 1 if self.spilled == 0: out.start_line() - out.emit_reload() + self.stack.emit_reload(out) + + def full_reload(self, out: CWriter) -> None: + if self.spilled == 0: + # Already spilled, presumably an explicit request + # to reload the stack from memory + out.start_line() + out.emit_str("// Explicit stack reload\n"); + self.stack.emit_fullreload(out, spilled=False) + else: + assert self.spilled > 0 + self.spilled -= 1 + implicit = True + self.stack.emit_fullreload(out) @staticmethod def for_uop(stack: Stack, uop: Uop, out: CWriter, check_liveness: bool = True) -> "Storage": + if stack.physical_sp is None: + raise StackError("stack pointer must be defined for uop") inputs: list[Local] = [] peeks: list[Local] = [] for input in reversed(uop.stack.inputs): @@ -632,7 +705,7 @@ def merge(self, other: "Storage", out: CWriter) -> None: def push_outputs(self) -> None: if self.spilled: - raise StackError(f"Unbalanced stack spills") + raise StackError(f"Too many stack spills") self.clear_inputs("at the end of the micro-op") if len(self.inputs) > self.peeks and self.check_liveness: raise StackError(f"Input variable '{self.inputs[-1].name}' is still live") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index d2fa749e1417f51..0d477cb83c5d436 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -9,8 +9,6 @@ Analysis, Instruction, Uop, - Label, - CodeSection, Part, analyze_files, Skip, @@ -24,13 +22,9 @@ write_header, type_and_null, Emitter, - TokenIterator, - always_true, - emit_to, ) from cwriter import CWriter from typing import TextIO -from lexer import Token from stack import Local, Stack, StackError, get_stack_effect, Storage DEFAULT_OUTPUT = ROOT / "Python/generated_cases.c.h" @@ -211,6 +205,7 @@ def generate_tier1_labels( storage = Storage(Stack(), [], [], 0, False) if label.spilled: storage.spilled = 1 + storage.stack.physical_sp = None emitter.emit_tokens(label, storage, None) emitter.emit("\n\n") diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 12da5bff254957c..b166b08b188deef 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -279,7 +279,7 @@ def generate_tier2( emitter = Tier2Emitter(out, analysis.labels, exit_depth) out.emit(f"case {uop.name}_r{inputs}{outputs}: {{\n") out.emit(f"CHECK_CURRENT_CACHED_VALUES({inputs});\n") - out.emit("assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());\n") + out.emit("ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);\n") declare_variables(uop, out) stack = Stack() stack.push_cache([f"_tos_cache{i}" for i in range(inputs)], out) @@ -287,7 +287,7 @@ def generate_tier2( reachable, stack = write_uop(uop, emitter, stack, outputs) out.start_line() if reachable: - out.emit("assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE());\n") + out.emit("ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__);\n") if not uop.properties.always_exits: out.emit("break;\n") out.start_line() diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index dfec0524cfe016d..fa53b205c4ff6af 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -43,10 +43,6 @@ PyDescr_TYPE PyWrapperFlag_KEYWORDS # cpython/fileobject.h Py_UniversalNewlineFgets -# cpython/ceval.h -PyUnstable_CopyPerfMapFile -PyUnstable_PerfTrampoline_CompileCode -PyUnstable_PerfTrampoline_SetPersistAfterFork # cpython/pyframe.h PyUnstable_EXECUTABLE_KINDS PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index f981f0bcaf89f0f..1c643caea98e3b5 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -173,12 +173,12 @@ def docstring_line_width(self) -> int: Pydoc adds indentation when displaying functions and methods. To keep the total width of within 80 characters, we use a - maximum of 76 characters for global functions and classes, - and 72 characters for methods. + maximum of 72 characters for global functions and classes, + and 68 characters for methods. """ if self.cls is not None and not self.kind.new_or_init: - return 72 - return 76 + return 68 + return 72 def __repr__(self) -> str: return f'<clinic.Function {self.name!r}>' diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 60f43b99c0f69dd..c8a914c22a9e137 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -279,6 +279,23 @@ def staticmethod_call(): for _ in range(1000 * WORK_SCALE): obj.my_staticmethod() + +class MyDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + def __set__(self, obj, value): + pass + +class MyClassWithDescriptor: + attr = MyDescriptor() + +@register_benchmark +def descriptor(): + obj = MyClassWithDescriptor() + for _ in range(1000 * WORK_SCALE): + obj.attr + @register_benchmark def deepcopy(): x = {'list': [1, 2], 'tuple': (1, None)} diff --git a/Tools/inspection/benchmark_external_inspection.py b/Tools/inspection/benchmark_external_inspection.py index fee3435496da0bd..b7aa0e5de7ed99b 100644 --- a/Tools/inspection/benchmark_external_inspection.py +++ b/Tools/inspection/benchmark_external_inspection.py @@ -4,7 +4,6 @@ import sys import contextlib import tempfile -import os import argparse from _colorize import get_colors, can_colorize @@ -151,6 +150,45 @@ def create_threads(n): time.sleep(0.05) ''' +ASYNC_CODE = '''\ +import asyncio +import contextlib +import math + +def compute_slice(seed): + result = 0.0 + for i in range(2000): + result += math.sin(seed + i) * math.sqrt(i + 1) + return result + +async def leaf_task(seed): + total = 0.0 + while True: + total += compute_slice(seed) + await asyncio.sleep(0) + +async def parent_task(seed): + child = asyncio.create_task(leaf_task(seed + 1000), name=f"leaf-{seed}") + try: + while True: + compute_slice(seed) + await asyncio.sleep(0.001) + finally: + child.cancel() + with contextlib.suppress(asyncio.CancelledError): + await child + +async def main(): + tasks = [ + asyncio.create_task(parent_task(i), name=f"parent-{i}") + for i in range(8) + ] + await asyncio.gather(*tasks) + +if __name__ == "__main__": + asyncio.run(main()) +''' + CODE_EXAMPLES = { "basic": { "code": CODE, @@ -164,10 +202,29 @@ def create_threads(n): "code": CODE_WITH_TONS_OF_THREADS, "description": "Tons of threads doing mixed CPU/IO work", }, + "asyncio": { + "code": ASYNC_CODE, + "description": "Asyncio tasks with active and awaited coroutine chains", + }, +} + +OPERATIONS = { + "stack_trace": { + "method": "get_stack_trace", + "label": "get_stack_trace()", + }, + "async_stack_trace": { + "method": "get_async_stack_trace", + "label": "get_async_stack_trace()", + }, + "all_awaited_by": { + "method": "get_all_awaited_by", + "label": "get_all_awaited_by()", + }, } -def benchmark(unwinder, duration_seconds=10, blocking=False): +def benchmark(unwinder, duration_seconds=10, blocking=False, operation="stack_trace"): """Benchmark mode - measure raw sampling speed for specified duration""" sample_count = 0 fail_count = 0 @@ -175,11 +232,14 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): start_time = time.perf_counter() end_time = start_time + duration_seconds total_attempts = 0 + operation_info = OPERATIONS[operation] + operation_method = getattr(unwinder, operation_info["method"]) colors = get_colors(can_colorize()) print( - f"{colors.BOLD_BLUE}Benchmarking sampling speed for {duration_seconds} seconds...{colors.RESET}" + f"{colors.BOLD_BLUE}Benchmarking {operation_info['label']} speed " + f"for {duration_seconds} seconds...{colors.RESET}" ) try: @@ -190,8 +250,8 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): if blocking: unwinder.pause_threads() try: - stack_trace = unwinder.get_stack_trace() - if stack_trace: + sample = operation_method() + if sample: sample_count += 1 finally: if blocking: @@ -239,6 +299,7 @@ def benchmark(unwinder, duration_seconds=10, blocking=False): (sample_count / total_attempts) * 100 if total_attempts > 0 else 0 ), "total_work_time": total_work_time, + "operation": operation_info["label"], "avg_work_time_us": ( (total_work_time / total_attempts) * 1e6 if total_attempts > 0 else 0 ), @@ -252,7 +313,7 @@ def print_benchmark_results(results): colors = get_colors(can_colorize()) print(f"\n{colors.BOLD_GREEN}{'='*60}{colors.RESET}") - print(f"{colors.BOLD_GREEN}get_stack_trace() Benchmark Results{colors.RESET}") + print(f"{colors.BOLD_GREEN}{results['operation']} Benchmark Results{colors.RESET}") print(f"{colors.BOLD_GREEN}{'='*60}{colors.RESET}") # Basic statistics @@ -329,6 +390,8 @@ def parse_arguments(): %(prog)s -d 60 # Run basic benchmark for 60 seconds %(prog)s --code deep_static # Run deep static call stack benchmark %(prog)s --code deep_static -d 30 # Run deep static benchmark for 30 seconds + %(prog)s --operation async_stack_trace + %(prog)s --operation all_awaited_by Available code examples: {examples_desc} @@ -348,8 +411,15 @@ def parse_arguments(): "--code", "-c", choices=list(CODE_EXAMPLES.keys()), - default="basic", - help="Code example to benchmark (default: basic)", + default=None, + help="Code example to benchmark (default: basic, or asyncio for async operations)", + ) + + parser.add_argument( + "--operation", + choices=list(OPERATIONS.keys()), + default="stack_trace", + help="Remote unwinder operation to benchmark (default: stack_trace)", ) parser.add_argument( @@ -365,7 +435,10 @@ def parse_arguments(): help="Stop all threads before sampling for consistent snapshots", ) - return parser.parse_args() + args = parser.parse_args() + if args.code is None: + args.code = "asyncio" if args.operation != "stack_trace" else "basic" + return args def create_target_process(temp_file, code_example="basic"): @@ -420,6 +493,9 @@ def main(): print( f"{colors.CYAN}Benchmark Duration:{colors.RESET} {colors.YELLOW}{args.duration}{colors.RESET} seconds" ) + print( + f"{colors.CYAN}Operation:{colors.RESET} {colors.GREEN}{OPERATIONS[args.operation]['label']}{colors.RESET}" + ) print( f"{colors.CYAN}Blocking Mode:{colors.RESET} {colors.GREEN if args.blocking else colors.YELLOW}{'enabled' if args.blocking else 'disabled'}{colors.RESET}" ) @@ -451,7 +527,12 @@ def main(): unwinder = _remote_debugging.RemoteUnwinder( process.pid, cache_frames=True, **kwargs ) - results = benchmark(unwinder, duration_seconds=args.duration, blocking=args.blocking) + results = benchmark( + unwinder, + duration_seconds=args.duration, + blocking=args.blocking, + operation=args.operation, + ) finally: cleanup_process(process, temp_file_path) diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index e2ae3d988fc7ac9..d4547dc8e8e3c12 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -2,7 +2,6 @@ import dataclasses import enum -import sys import typing import _schema diff --git a/Tools/jit/template.c b/Tools/jit/template.c index afdd9b77e7c7ffa..f1e2e25aaecdada 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -1,5 +1,11 @@ + #include "Python.h" +#ifndef NDEBUG +#undef assert +#define assert(TEST) ((TEST) ? 0 : _Py_jit_assertion_failure(__LINE__)) +#endif + #include "pycore_backoff.h" #include "pycore_call.h" #include "pycore_cell.h" @@ -36,6 +42,10 @@ #include "jit.h" +#ifndef NDEBUG +#undef assert +#define assert(TEST) ((TEST) ? 0 : _Py_jit_assertion_failure(__LINE__)) +#endif #undef CURRENT_OPERAND0_64 #define CURRENT_OPERAND0_64() (_operand0_64) @@ -115,7 +125,7 @@ do { \ #define TIER_TWO 2 #ifdef Py_DEBUG -#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_assert_within_stack_bounds(frame, stack_pointer, (F), (L)) +#define ASSERT_WITHIN_STACK_BOUNDS(F, L) _Py_jit_assert_within_stack_bounds(frame, stack_pointer, (L)) #else #define ASSERT_WITHIN_STACK_BOUNDS(F, L) (void)0 #endif diff --git a/Tools/msi/common.wxs b/Tools/msi/common.wxs index 54fa749ab17cdd0..73da474e4181f11 100644 --- a/Tools/msi/common.wxs +++ b/Tools/msi/common.wxs @@ -75,6 +75,12 @@ </Fragment> <!-- Top-level directories --> + <Fragment> + <DirectoryRef Id="InstallDirectory"> + <Directory Id="abi3t_compat" Name="abi3t-compat" /> + </DirectoryRef> + </Fragment> + <Fragment> <DirectoryRef Id="InstallDirectory"> <Directory Id="DLLs" Name="DLLs"> diff --git a/Tools/msi/core/core_files.wxs b/Tools/msi/core/core_files.wxs index 145e1471247aa1e..8b21501078ea2e6 100644 --- a/Tools/msi/core/core_files.wxs +++ b/Tools/msi/core/core_files.wxs @@ -2,6 +2,9 @@ <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"> <Fragment> <ComponentGroup Id="core_dll"> + <Component Id="python_abi3tcompat.dll" Directory="abi3t_compat" Guid="*"> + <File Id="python_abi3tcompat.dll" Name="python$(var.MajorVersionNumber)t.dll" KeyPath="yes" /> + </Component> <Component Id="python_stable.dll" Directory="InstallDirectory" Guid="*"> <File Id="python_stable.dll" Name="python$(var.MajorVersionNumber).dll" KeyPath="yes" /> </Component> @@ -19,6 +22,9 @@ </Fragment> <Fragment> <ComponentGroup Id="core_dll_d"> + <Component Id="python_abi3tcompat_d.dll" Directory="abi3t_compat" Guid="*"> + <File Id="python_abi3tcompat_d.dll" Name="python$(var.MajorVersionNumber)t_d.dll" KeyPath="yes" /> + </Component> <Component Id="python_stable_d.dll" Directory="InstallDirectory" Guid="*"> <File Id="python_stable_d.dll" Name="python$(var.MajorVersionNumber)_d.dll" KeyPath="yes" /> </Component> diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 21f9c848cc6be58..a9039d03f5f6fa1 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -13,6 +13,9 @@ <Component Id="libs_python3.lib" Directory="libs" Guid="*"> <File Id="libs_python_stable.lib" Name="python$(var.MajorVersionNumber).lib" KeyPath="yes" /> </Component> + <Component Id="libs_python3t.lib" Directory="libs" Guid="*"> + <File Id="libs_python_abi3tcompat.lib" Name="python$(var.MajorVersionNumber)t.lib" KeyPath="yes" /> + </Component> <Component Id="libs_python.lib" Directory="libs" Guid="*"> <File Id="libs_python.lib" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber).lib" KeyPath="yes" /> </Component> @@ -24,6 +27,9 @@ <Component Id="libs_python3_d.lib" Directory="libs" Guid="*"> <File Id="libs_python_stable_d.lib" Name="python$(var.MajorVersionNumber)_d.lib" /> </Component> + <Component Id="libs_python3t_d.lib" Directory="libs" Guid="*"> + <File Id="libs_python_abi3tcompat_d.lib" Name="python$(var.MajorVersionNumber)t_d.lib" /> + </Component> <Component Id="libs_python_d.lib" Directory="libs" Guid="*"> <File Id="libs_python_d.lib" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)_d.lib" /> </Component> diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index 0707e77b5e9ab2e..fdbcaea38eb3175 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -29,7 +29,7 @@ <ComponentGroup Id="freethreaded_exe"> <Component Id="freethreaded_python.exe" Directory="InstallDirectory" Guid="$(var.FreethreadedPythonExeComponentGuid)"> - <File Name="python$(var.ShortVersion)t.exe" KeyPath="yes" /> + <File Name="python$(var.ShortVersion)t.exe" Source="!(bindpath.build_t)\python.exe" KeyPath="yes" /> <RegistryKey Root="HKMU" Key="[FREETHREADED_REGISTRYKEY]"> <RegistryValue Key="InstallPath" Type="string" Value="[InstallDirectory]" KeyPath="no" /> @@ -37,68 +37,72 @@ </RegistryKey> </Component> <Component Id="freethreaded_pythonw.exe" Directory="InstallDirectory" Guid="$(var.FreethreadedPythonwExeComponentGuid)"> - <File Name="pythonw$(var.ShortVersion)t.exe" KeyPath="yes" /> + <File Name="pythonw$(var.ShortVersion)t.exe" Source="!(bindpath.build_t)\pythonw.exe" KeyPath="yes" /> <RegistryKey Root="HKMU" Key="[FREETHREADED_REGISTRYKEY]"> <RegistryValue Key="InstallPath" Name="WindowedExecutablePath" Type="string" Value="[#pythonw$(var.ShortVersion)t.exe]" KeyPath="no" /> </RegistryKey> </Component> <Component Id="freethreaded_python_stable.dll" Directory="InstallDirectory" Guid="*"> - <File Id="freethreaded_python_stable.dll" Name="python$(var.MajorVersionNumber)t.dll" KeyPath="yes" /> + <File Id="freethreaded_python_stable.dll" Name="python$(var.MajorVersionNumber)t.dll" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)t.dll" KeyPath="yes" /> </Component> <Component Id="freethreaded_python.dll" Directory="InstallDirectory" Guid="*"> - <File Id="freethreaded_python.dll" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.dll" KeyPath="yes" /> + <File Id="freethreaded_python.dll" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.dll" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.dll" KeyPath="yes" /> </Component> <Component Id="freethreaded_python_stable.lib" Directory="libs" Guid="*"> - <File Id="freethreaded_python_stable.lib" Name="python$(var.MajorVersionNumber)t.lib" KeyPath="yes" /> + <File Id="freethreaded_python_stable.lib" Name="python$(var.MajorVersionNumber)t.lib" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)t.lib" KeyPath="yes" /> </Component> <Component Id="freethreaded_python.lib" Directory="libs" Guid="*"> - <File Id="freethreaded_python.lib" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.lib" KeyPath="yes" /> + <File Id="freethreaded_python.lib" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.lib" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.lib" KeyPath="yes" /> </Component> </ComponentGroup> </Fragment> <Fragment> <ComponentGroup Id="freethreaded_symbols"> <Component Id="freethreaded_python_dll.pdb" Directory="InstallDirectory" Guid="*"> - <File Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.pdb" KeyPath="yes" /> + <File Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.pdb" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t.pdb" KeyPath="yes" /> </Component> + <!-- + Renaming the PDB like this will break automatic connection by debuggers, + but short of making people stop using this installer there's no real option. + --> <Component Id="freethreaded_python.pdb" Directory="InstallDirectory" Guid="*"> - <File Name="python$(var.ShortVersion)t.pdb" /> + <File Name="python$(var.ShortVersion)t.pdb" Source="!(bindpath.build_t)\python.pdb" /> </Component> <Component Id="freethreaded_pythonw.pdb" Directory="InstallDirectory" Guid="*"> - <File Name="pythonw$(var.ShortVersion)t.pdb" /> + <File Name="pythonw$(var.ShortVersion)t.pdb" Source="!(bindpath.build_t)\pythonw.pdb" /> </Component> </ComponentGroup> </Fragment> <Fragment> <ComponentGroup Id="freethreaded_dll_d"> <Component Id="freethreaded_python_stable_d.dll" Directory="InstallDirectory" Guid="*"> - <File Id="freethreaded_python_stable_d.dll" Name="python$(var.MajorVersionNumber)t_d.dll" KeyPath="yes" /> + <File Id="freethreaded_python_stable_d.dll" Name="python$(var.MajorVersionNumber)t_d.dll" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)t_d.dll" KeyPath="yes" /> </Component> <Component Id="freethreaded_python_d.dll" Directory="InstallDirectory" Guid="*"> - <File Id="freethreaded_python_d.dll" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.dll" KeyPath="yes" /> - <File Id="freethreaded_python_d.pdb" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.pdb" KeyPath="no" /> + <File Id="freethreaded_python_d.dll" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.dll" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.dll" KeyPath="yes" /> + <File Id="freethreaded_python_d.pdb" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.pdb" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.pdb" KeyPath="no" /> </Component> <Component Id="freethreaded_python_stable_d.lib" Directory="libs" Guid="*"> - <File Id="freethreaded_python_stable_d.lib" Name="python$(var.MajorVersionNumber)t_d.lib" KeyPath="yes" /> + <File Id="freethreaded_python_stable_d.lib" Name="python$(var.MajorVersionNumber)t_d.lib" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)t_d.lib" KeyPath="yes" /> </Component> <Component Id="freethreaded_python_d.lib" Directory="libs" Guid="*"> - <File Id="freethreaded_python_d.lib" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.lib" KeyPath="yes" /> + <File Id="freethreaded_python_d.lib" Name="python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.lib" Source="!(bindpath.build_t)\python$(var.MajorVersionNumber)$(var.MinorVersionNumber)t_d.lib" KeyPath="yes" /> </Component> </ComponentGroup> </Fragment> <Fragment> <ComponentGroup Id="freethreaded_exe_d"> <Component Id="freethreaded_python_d.exe" Directory="InstallDirectory" Guid="*"> - <File Name="python$(var.ShortVersion)t_d.exe" /> + <File Name="python$(var.ShortVersion)t_d.exe" Source="!(bindpath.build_t)python_d.exe" /> </Component> <Component Id="freethreaded_python_d.pdb" Directory="InstallDirectory" Guid="*"> - <File Name="python$(var.ShortVersion)t_d.pdb" /> + <File Name="python$(var.ShortVersion)t_d.pdb" Source="!(bindpath.build_t)python_d.pdb" /> </Component> <Component Id="freethreaded_pythonw_d.exe" Directory="InstallDirectory" Guid="*"> - <File Name="pythonw$(var.ShortVersion)t_d.exe" /> + <File Name="pythonw$(var.ShortVersion)t_d.exe" Source="!(bindpath.build_t)pythonw_d.exe" /> </Component> <Component Id="freethreaded_pythonw_d.pdb" Directory="InstallDirectory" Guid="*"> - <File Name="pythonw$(var.ShortVersion)t_d.pdb" /> + <File Name="pythonw$(var.ShortVersion)t_d.pdb" Source="!(bindpath.build_t)pythonw_d.pdb" /> </Component> </ComponentGroup> </Fragment> @@ -111,16 +115,16 @@ <?foreach ext in $(var.exts)?> <Component Id="freethreaded_$(var.ext).pyd" Directory="DLLs" Guid="*"> - <File Name="$(var.ext)$(var.FreethreadedPydTag).pyd" KeyPath="yes" /> + <File Name="$(var.ext)$(var.FreethreadedPydTag).pyd" Source="!(bindpath.build_t)\$(var.ext)$(var.FreethreadedPydTag).pyd" KeyPath="yes" /> </Component> <?endforeach ?> <Component Id="venvlaunchert.exe" Directory="Lib_venv_scripts_nt__freethreaded" Guid="*"> - <File Name="venvlaunchert.exe" KeyPath="yes" /> + <File Name="venvlaunchert.exe" Source="!(bindpath.build_t)\venvlaunchert.exe" KeyPath="yes" /> </Component> <Component Id="venvwlaunchert.exe" Directory="Lib_venv_scripts_nt__freethreaded" Guid="*"> - <File Name="venvwlaunchert.exe" KeyPath="yes" /> + <File Name="venvwlaunchert.exe" Source="!(bindpath.build_t)\venvwlaunchert.exe" KeyPath="yes" /> </Component> </ComponentGroup> </Fragment> @@ -132,16 +136,16 @@ <?foreach ext in $(var.exts)?> <Component Id="freethreaded_$(var.ext).pdb" Directory="DLLs" Guid="*"> - <File Name="$(var.ext)$(var.FreethreadedPydTag).pdb" /> + <File Name="$(var.ext)$(var.FreethreadedPydTag).pdb" Source="!(bindpath.build_t)\$(var.ext)$(var.FreethreadedPydTag).pdb" /> </Component> <?endforeach ?> <Component Id="venvlaunchert.pdb" Directory="Lib_venv_scripts_nt__freethreaded" Guid="*"> - <File Name="venvlaunchert.pdb" KeyPath="yes" /> + <File Name="venvlaunchert.pdb" Source="!(bindpath.build_t)\venvlaunchert.pdb" KeyPath="yes" /> </Component> <Component Id="venvwlaunchert.pdb" Directory="Lib_venv_scripts_nt__freethreaded" Guid="*"> - <File Name="venvwlaunchert.pdb" KeyPath="yes" /> + <File Name="venvwlaunchert.pdb" Source="!(bindpath.build_t)\venvwlaunchert.pdb" KeyPath="yes" /> </Component> </ComponentGroup> </Fragment> @@ -151,21 +155,21 @@ <?foreach ext in $(var.exts)?> <Component Id="freethreaded_$(var.ext)_d.pyd" Directory="DLLs" Guid="*"> - <File Name="$(var.ext)_d$(var.FreethreadedPydTag).pyd" /> + <File Name="$(var.ext)_d$(var.FreethreadedPydTag).pyd" Source="!(bindpath.build_t)\$(var.ext)_d$(var.FreethreadedPydTag).pyd" /> </Component> <Component Id="freethreaded_$(var.ext)_d.pdb" Directory="DLLs" Guid="*"> - <File Name="$(var.ext)_d$(var.FreethreadedPydTag).pdb" /> + <File Name="$(var.ext)_d$(var.FreethreadedPydTag).pdb" Source="!(bindpath.build_t)\$(var.ext)_d$(var.FreethreadedPydTag).pdb" /> </Component> <?endforeach ?> <Component Id="venvlaunchert_d.exe" Directory="Lib_venv_scripts_nt__freethreaded" Guid="*"> - <File Name="venvlaunchert_d.exe" KeyPath="yes" /> - <File Name="venvlaunchert_d.pdb" /> + <File Name="venvlaunchert_d.exe" Source="!(bindpath.build_t)\venvlaunchert_d.exe" KeyPath="yes" /> + <File Name="venvlaunchert_d.pdb" Source="!(bindpath.build_t)\venvlaunchert_d.pdb" /> </Component> <Component Id="venvwlaunchert_d.exe" Directory="Lib_venv_scripts_nt__freethreaded" Guid="*"> - <File Name="venvwlaunchert_d.exe" KeyPath="yes" /> - <File Name="venvwlaunchert_d.pdb" /> + <File Name="venvwlaunchert_d.exe" Source="!(bindpath.build_t)\venvwlaunchert_d.exe" KeyPath="yes" /> + <File Name="venvwlaunchert_d.pdb" Source="!(bindpath.build_t)\venvwlaunchert_d.pdb" /> </Component> </ComponentGroup> </Fragment> diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props index 372c4823bce07f3..097af60715448f1 100644 --- a/Tools/msi/msi.props +++ b/Tools/msi/msi.props @@ -122,6 +122,30 @@ <LinkerBindInputPaths Include="$(PGOBuildPath);$(BuildPath)"> <BindName></BindName> </LinkerBindInputPaths> + <!-- + This looks repetitive, but we need to make sure that we override any + environment variable that changes the paths for free-threading so that + the binds are correct. Otherwise, we'd rely on defaults. + --> + <LinkerBindInputPaths Include="$(BuildPath32)" Condition="$(Platform) == 'x86'"> + <BindName>build</BindName> + </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPath64)" Condition="$(Platform) == 'x64'"> + <BindName>build</BindName> + </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPathARM64)" Condition="$(Platform) == 'ARM64'"> + <BindName>build</BindName> + </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPath32t)" Condition="$(Platform) == 'x86'"> + <BindName>build_t</BindName> + </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPath64t)" Condition="$(Platform) == 'x64'"> + <BindName>build_t</BindName> + </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPathARM64t)" Condition="$(Platform) == 'ARM64'"> + <BindName>build_t</BindName> + </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(PySourcePath)"> <BindName>src</BindName> </LinkerBindInputPaths> @@ -131,6 +155,7 @@ <LinkerBindInputPaths Include="$(CRTRedist)"> <BindName>redist</BindName> </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPath32)"> <BindName>build32</BindName> </LinkerBindInputPaths> @@ -140,6 +165,15 @@ <LinkerBindInputPaths Include="$(BuildPathARM64)"> <BindName>buildarm64</BindName> </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPath32t)"> + <BindName>build32t</BindName> + </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPath64t)"> + <BindName>build64t</BindName> + </LinkerBindInputPaths> + <LinkerBindInputPaths Include="$(BuildPathARM64t)"> + <BindName>buildarm64t</BindName> + </LinkerBindInputPaths> </ItemGroup> <Target Name="_ValidateMsiProps" BeforeTargets="PrepareForBuild"> diff --git a/Tools/msi/tcltk/tcltk_files.wxs b/Tools/msi/tcltk/tcltk_files.wxs index 5dad7c98d4f048a..7c7784741d9178d 100644 --- a/Tools/msi/tcltk/tcltk_files.wxs +++ b/Tools/msi/tcltk/tcltk_files.wxs @@ -10,11 +10,14 @@ <Component Id="_tkinter.lib" Directory="libs" Guid="*"> <File Name="_tkinter.lib" KeyPath="yes" /> </Component> - <Component Id="tcl86t.dll" Directory="DLLs" Guid="*"> - <File Name="tcl86t.dll" KeyPath="yes" /> + <Component Id="tcl90.dll" Directory="DLLs" Guid="*"> + <File Name="tcl90.dll" KeyPath="yes" /> </Component> - <Component Id="tk86t.dll" Directory="DLLs" Guid="*"> - <File Name="tk86t.dll" KeyPath="yes" /> + <Component Id="tcl9tk90.dll" Directory="DLLs" Guid="*"> + <File Name="tcl9tk90.dll" KeyPath="yes" /> + </Component> + <Component Id="libtommath.dll" Directory="DLLs" Guid="*"> + <File Name="libtommath.dll" KeyPath="yes" /> </Component> <Component Id="zlib1.dll" Directory="DLLs" Guid="*"> <File Name="zlib1.dll" KeyPath="yes" /> diff --git a/Tools/msi/testrelease.bat b/Tools/msi/testrelease.bat index 02bcca943cf79b4..db98f690151196c 100644 --- a/Tools/msi/testrelease.bat +++ b/Tools/msi/testrelease.bat @@ -88,9 +88,7 @@ exit /B 0 ) @if not errorlevel 1 ( @echo Testing Tcl/tk - @set TCL_LIBRARY=%~2\Python\tcl\tcl8.6 "%~2\Python\python.exe" -m test -uall -v test_ttk_guionly test_tk test_idle > "%~2\tcltk.txt" 2>&1 - @set TCL_LIBRARY= ) @set EXITCODE=%ERRORLEVEL% diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 44a58581686a4f7..bc3657067a8da2e 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -145,10 +145,15 @@ def compile_c_extension( str(MOD_DIR.parent.parent.parent / "Parser" / "lexer"), str(MOD_DIR.parent.parent.parent / "Parser" / "tokenizer"), ] + library_dirs: list[str] = [] if sys.platform == "win32": # HACK: The location of pyconfig.h has moved within our build, and # setuptools hasn't updated for it yet. So add the path manually for now - include_dirs.append(pathlib.Path(sysconfig.get_config_h_filename()).parent) + include_dirs.append(str(pathlib.Path(sysconfig.get_config_h_filename()).parent)) + if sysconfig.is_python_build(): + # HACK: Our output directory for free-threaded builds has moved, and so + # tests running in-tree require our sys.executable directory for libs + library_dirs.append(str(pathlib.Path(sys.executable).parent)) extension = Extension( extension_name, sources=[generated_source_path], @@ -161,6 +166,7 @@ def compile_c_extension( fixup_build_ext(cmd) cmd.build_lib = str(source_file_path.parent) cmd.include_dirs = include_dirs + cmd.library_dirs = library_dirs if build_dir: cmd.build_temp = build_dir cmd.ensure_finalized() diff --git a/Tools/picklebench/memory_dos_impact.py b/Tools/picklebench/memory_dos_impact.py index 3bad6586c469437..0a7cef8668565c8 100755 --- a/Tools/picklebench/memory_dos_impact.py +++ b/Tools/picklebench/memory_dos_impact.py @@ -24,7 +24,6 @@ import argparse import gc -import io import json import os import pickle @@ -32,11 +31,10 @@ import struct import subprocess import sys -import tempfile import tracemalloc from pathlib import Path from time import perf_counter -from typing import Any, Dict, List, Tuple, Optional +from typing import Any, Dict, List, Tuple # Configuration diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index af5cbaa7689f33d..46381ea58a12382 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.19.1 +mypy==2.1.0 # needed for peg_generator: -types-psutil==7.2.2.20260130 -types-setuptools==82.0.0.20260210 +types-psutil==7.2.2.20260508 +types-setuptools==82.0.0.20260508 diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index ad3e4b6596ed7ae..1a213187b897d1d 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -46,15 +46,15 @@ "1.1.1w", "3.1.8", "3.2.6", + "3.3.7", ] OPENSSL_RECENT_VERSIONS = [ - "3.0.20", - "3.3.7", - "3.4.5", - "3.5.6", - "3.6.2", - "4.0.0", + "3.0.21", + "3.4.6", + "3.5.7", + "3.6.3", + "4.0.1", # See make_ssl_data.py for notes on adding a new version. ] @@ -65,7 +65,7 @@ ] AWSLC_RECENT_VERSIONS = [ - "1.68.0", + "5.0.0", ] # store files in ../multissl diff --git a/Tools/unicode/gen_expat_table.py b/Tools/unicode/gen_expat_table.py new file mode 100644 index 000000000000000..ff08fb4c87dab39 --- /dev/null +++ b/Tools/unicode/gen_expat_table.py @@ -0,0 +1,107 @@ +import sys + + +def info(msg, *args): + print(msg % args, file=sys.stderr) + +def fail(msg, *args): + info(msg, *args) + sys.exit(2) + +# ASCII characters that can appear in a well-formed XML document +# except the characters "$@\^`{}~". +compulsory_chars = (set(b'\t\n\r') | set(range(32, 127))) - set(br'$@\^`{}~') + +def check_compatibility(encoding): + for i in compulsory_chars: + c = chr(i) + b = bytes([i]) + try: + decoded = b.decode(encoding) + except UnicodeDecodeError: + info('Cannot decode byte %#04x (%a)', i, c) + return False + if decoded != c: + info('Incompatible encoding: %#04x (%a) -> %a', i, c, decoded) + return False + return True + +def create_table(encoding): + mapping = [-1] * 256 + + non_bmp = None + for i in range(0x110000): + char = chr(i) + try: + encoded = char.encode(encoding) + except UnicodeEncodeError: + continue + if i >= 0x10000 and non_bmp is None: + non_bmp = i + info('Non-BMP character: %r (U+%04X)', char, i) + length = len(encoded) + k = encoded[0] + v = mapping[k] + if length == 1: + if v == -1: + mapping[k] = i + elif v < 0: + fail('Ambiguous mapping for %#04x: ' + '%r (U+%04X) and %d-byte sequence', + k, c, i, -v) + else: + info('Ambiguous mapping for %#04x: ' + '%r (U+%04X) and %r (U+%04X)', + k, chr(v), v, char, i) + else: + if length > 4: + fail('Too long encoding for %r (U+%04X): %r', + char, i, encoded) + if v == -1: + mapping[k] = -length + elif v != -length: + if v < 0: + fail('Ambiguous mapping for %#04x: ' + '%d-byte sequence and %d-byte sequence %r', + k, -v, length, encoded) + else: + fail('Ambiguous mapping for %#04x: ' + '%r (U+%04X) and %d-byte sequence %r', + k, chr(v), v, length, encoded) + return mapping + +def print_table(mapping): + print(' '*8 + '_expat_decoding_table=(', end='') + if mapping[:128] == list(range(128)): + print('*range(128),') + start = 128 + else: + print() + start = 0 + line = '' + pos = 0 + i = start + while True: + if i % 8 == 0: + if len(line) > 68: + print(' '*12 + line[:pos].rstrip()) + line = line[pos:] + pos = len(line) + if i == 256: + break + v = mapping[i] + if v >= 0: + v = hex(v) + line += f'{v}, ' + i += 1 + print(' '*12 + line.rstrip().rstrip(',') + '),') + + +encoding = sys.argv[1] + +if not check_compatibility(encoding): + print(' '*8 + '_expat_decoding_table=False,') + sys.exit(1) + +mapping = create_table(encoding) +print_table(mapping) diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index 5db850ca2d1f0c0..76283d6b794a0b0 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -342,15 +342,21 @@ def makeunicodedata(unicode, trace): fprint("#define TOTAL_FIRST",total_first) fprint("#define TOTAL_LAST",total_last) fprint("struct reindex{int start;short count,index;};") + # The reindex tables are read only by find_nfc_index(), which scans + # forward while .start <= code. The trailing sentinel's .start must + # exceed every codepoint (so the scan stops with a single comparison) + # and fit the signed int .start field. + nfc_sentinel = 0x7fffffff + assert sys.maxunicode < nfc_sentinel <= 0x7fffffff fprint("static struct reindex nfc_first[] = {") for start,end in comp_first_ranges: fprint(" { %d, %d, %d}," % (start,end-start,comp_first[start])) - fprint(" {0,0,0}") + fprint(" {0x%x, 0, 0}" % nfc_sentinel) fprint("};\n") fprint("static struct reindex nfc_last[] = {") for start,end in comp_last_ranges: fprint(" { %d, %d, %d}," % (start,end-start,comp_last[start])) - fprint(" {0,0,0}") + fprint(" {0x%x, 0, 0}" % nfc_sentinel) fprint("};\n") # FIXME: <fl> the following tables could be made static, and diff --git a/configure b/configure index 734aa3a6a721d1e..e96b87989793a8f 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.72 for python 3.15. +# Generated by GNU Autoconf 2.72 for python 3.16. # # Report bugs to <https://github.com/python/cpython/issues/>. # @@ -604,8 +604,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='python' PACKAGE_TARNAME='python' -PACKAGE_VERSION='3.15' -PACKAGE_STRING='python 3.15' +PACKAGE_VERSION='3.16' +PACKAGE_STRING='python 3.16' PACKAGE_BUGREPORT='https://github.com/python/cpython/issues/' PACKAGE_URL='' @@ -642,11 +642,14 @@ ac_includes_default="\ #endif" ac_header_c_list= +ac_func_c_list= ac_subst_vars='LTLIBOBJS MODULE_BLOCK JIT_SHIM_BUILD_O JIT_SHIM_O JIT_STENCILS_H +MODULE_XXLIMITED_3_13_FALSE +MODULE_XXLIMITED_3_13_TRUE MODULE_XXLIMITED_35_FALSE MODULE_XXLIMITED_35_TRUE MODULE_XXLIMITED_FALSE @@ -826,6 +829,7 @@ MODULE__IO_FALSE MODULE__IO_TRUE MODULE_BUILDTYPE _PYTHREAD_NAME_MAXLEN +BUILD_DETAILS TEST_MODULES OPENSSL_LDFLAGS OPENSSL_LIBS @@ -891,7 +895,6 @@ TCLTK_LIBS TCLTK_CFLAGS LIBSQLITE3_LIBS LIBSQLITE3_CFLAGS -LIBMPDEC_INTERNAL LIBMPDEC_LIBS LIBMPDEC_CFLAGS MODULE__CTYPES_MALLOC_CLOSURE @@ -1126,7 +1129,6 @@ with_hash_algorithm with_tzpath with_libs with_system_expat -with_system_libmpdec with_decimal_contextvar enable_loadable_sqlite_extensions with_dbmliborder @@ -1154,6 +1156,7 @@ with_openssl_rpath with_ssl_default_suites with_builtin_hashlib_hashes enable_test_modules +with_build_details_suffix ' ac_precious_vars='build_alias host_alias @@ -1750,7 +1753,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -'configure' configures python 3.15 to adapt to many kinds of systems. +'configure' configures python 3.16 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1816,7 +1819,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.15:";; + short | recursive ) echo "Configuration of python 3.16:";; esac cat <<\_ACEOF @@ -1875,9 +1878,9 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.15 + --with-build-python=python3.16 path to build python binary for cross compiling - (default: _bootstrap_python or python3.15) + (default: _bootstrap_python or python3.16) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) @@ -1936,9 +1939,6 @@ Optional Packages: --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) - --with-system-libmpdec build _decimal module using an installed mpdecimal - library, see Doc/library/decimal.rst (default is - yes) --with-decimal-contextvar build _decimal module using a coroutine-local rather than a thread-local context (default is yes) @@ -1992,6 +1992,10 @@ Optional Packages: --with-builtin-hashlib-hashes=md5,sha1,sha2,sha3,blake2 builtin hash modules, md5, sha1, sha2, sha3 (with shake), blake2 + --with-build-details-suffix= + rename build-details.json to permit multiple + colocated Python installs; optionally specify a + custom suffix (default: no) Some influential environment variables: PKG_CONFIG path to pkg-config utility @@ -2133,7 +2137,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.15 +python configure 3.16 generated by GNU Autoconf 2.72 Copyright (C) 2023 Free Software Foundation, Inc. @@ -2811,7 +2815,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.15, which was +It was created by python $as_me 3.16, which was generated by GNU Autoconf 2.72. Invocation command line was $ $0$ac_configure_args_raw @@ -3418,6 +3422,13 @@ as_fn_append ac_header_c_list " sys/types.h sys_types_h HAVE_SYS_TYPES_H" as_fn_append ac_header_c_list " unistd.h unistd_h HAVE_UNISTD_H" as_fn_append ac_header_c_list " wchar.h wchar_h HAVE_WCHAR_H" as_fn_append ac_header_c_list " minix/config.h minix_config_h HAVE_MINIX_CONFIG_H" +as_fn_append ac_func_c_list " acospi HAVE_ACOSPI" +as_fn_append ac_func_c_list " asinpi HAVE_ASINPI" +as_fn_append ac_func_c_list " atanpi HAVE_ATANPI" +as_fn_append ac_func_c_list " atan2pi HAVE_ATAN2PI" +as_fn_append ac_func_c_list " cospi HAVE_COSPI" +as_fn_append ac_func_c_list " sinpi HAVE_SINPI" +as_fn_append ac_func_c_list " tanpi HAVE_TANPI" # Auxiliary files required by this configure script. ac_aux_files="install-sh config.guess config.sub" @@ -3599,7 +3610,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then # If we're building out-of-tree, we need to make sure the following # resources get picked up before their $srcdir counterparts. - # Objects/ -> typeslots.inc + # Objects/ -> slots_generated.c # Include/ -> Python.h # (A side effect of this is that these resources will automatically be # regenerated when building out-of-tree, regardless of whether or not @@ -3846,7 +3857,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.16 python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -3922,7 +3933,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.15 +VERSION=3.16 # Version number of Python's own shared library file. @@ -4154,7 +4165,7 @@ then ac_sys_system=Linux ;; *-*-cygwin*) - ac_sys_system=Cygwin + ac_sys_system=CYGWIN ;; *-apple-ios*) ac_sys_system=iOS @@ -6643,7 +6654,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6692,7 +6703,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6724,7 +6735,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="g++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6753,7 +6764,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6802,7 +6813,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6834,7 +6845,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="c++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6863,7 +6874,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6912,7 +6923,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6944,7 +6955,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="clang++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6973,7 +6984,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7022,7 +7033,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7054,7 +7065,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="icpc" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -7786,7 +7797,7 @@ printf "%s\n" "#define Py_ENABLE_SHARED 1" >>confdefs.h CYGWIN*) LDLIBRARY='libpython$(LDVERSION).dll.a' BLDLIBRARY='-L. -lpython$(LDVERSION)' - DLLLIBRARY='libpython$(LDVERSION).dll' + DLLLIBRARY='cygpython$(LDVERSION).dll' ;; SunOS*) LDLIBRARY='libpython$(LDVERSION).so' @@ -7845,7 +7856,7 @@ else # shared is disabled case $ac_sys_system in CYGWIN*) BLDLIBRARY='$(LIBRARY)' - LDLIBRARY='libpython$(LDVERSION).dll.a' + LDLIBRARY='libpython$(LDVERSION).a' ;; esac fi @@ -9590,7 +9601,7 @@ printf "%s\n" "$BOLT_INSTRUMENT_FLAGS" >&6; } printf %s "checking BOLT_APPLY_FLAGS... " >&6; } if test -z "${BOLT_APPLY_FLAGS}" then - BOLT_APPLY_FLAGS=" ${BOLT_COMMON_FLAGS} -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot " + BOLT_APPLY_FLAGS=" ${BOLT_COMMON_FLAGS} -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions -icf=0 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot " fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $BOLT_APPLY_FLAGS" >&5 @@ -9788,10 +9799,12 @@ fi as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES" - as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" as_fn_append LINKFORSHARED " -sTEXTDECODER=2" + as_fn_append LDFLAGS_NODIST " -Wl,--wrap=getentropy" + if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -9864,6 +9877,65 @@ fi ;; esac +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for thread stack size" >&5 +printf %s "checking for thread stack size... " >&6; } +if test ${ac_cv_thread_stack_size+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + cat > conftest.c <<EOF +#include <pthread.h> + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_thread_stack_size" >&5 +printf "%s\n" "$ac_cv_thread_stack_size" >&6; } + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + # Stack size used by Python/ceval.c to set Py_C_STACK_SIZE + +printf "%s\n" "#define _Py_LINKER_THREAD_STACK_SIZE $ac_cv_thread_stack_size" >>confdefs.h + + fi +fi + case $enable_wasm_dynamic_linking in #( yes) : ac_cv_func_dlopen=yes ;; #( @@ -10343,6 +10415,157 @@ else case e in #( esac fi + case $host_cpu in #( + arm|armv*) : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -marm" >&5 +printf %s "checking whether C compiler accepts -marm... " >&6; } +if test ${ax_cv_check_cflags__Werror__marm+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -marm" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__marm=yes +else case e in #( + e) ax_cv_check_cflags__Werror__marm=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__marm" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__marm" >&6; } +if test "x$ax_cv_check_cflags__Werror__marm" = xyes +then : + + frame_pointer_cflags="$frame_pointer_cflags -marm" + +else case e in #( + e) : ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mno-thumb" >&5 +printf %s "checking whether C compiler accepts -mno-thumb... " >&6; } +if test ${ax_cv_check_cflags__Werror__mno_thumb+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -mno-thumb" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__mno_thumb=yes +else case e in #( + e) ax_cv_check_cflags__Werror__mno_thumb=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__mno_thumb" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__mno_thumb" >&6; } +if test "x$ax_cv_check_cflags__Werror__mno_thumb" = xyes +then : + + frame_pointer_cflags="$frame_pointer_cflags -mno-thumb" + +else case e in #( + e) : ;; +esac +fi + + ;; #( + *) : + ;; +esac + case $host_cpu in #( + powerpc64le) : + + frame_pointer_cflags="" + ;; #( + *) : + ;; +esac + case $host_cpu in #( + s390*) : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mbackchain" >&5 +printf %s "checking whether C compiler accepts -mbackchain... " >&6; } +if test ${ax_cv_check_cflags__Werror__mbackchain+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -mbackchain" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__mbackchain=yes +else case e in #( + e) ax_cv_check_cflags__Werror__mbackchain=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__mbackchain" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__mbackchain" >&6; } +if test "x$ax_cv_check_cflags__Werror__mbackchain" = xyes +then : + + frame_pointer_cflags="-mbackchain" + +else case e in #( + e) : ;; +esac +fi + + ;; #( + *) : + ;; +esac else case e in #( e) : ;; @@ -10351,6 +10574,9 @@ fi if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then BASECFLAGS="$frame_pointer_cflags $BASECFLAGS" + +printf "%s\n" "#define _Py_WITH_FRAME_POINTERS 1" >>confdefs.h + fi CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -11878,6 +12104,12 @@ if test "x$ac_cv_header_sys_syscall_h" = xyes then : printf "%s\n" "#define HAVE_SYS_SYSCALL_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "sys/sysctl.h" "ac_cv_header_sys_sysctl_h" "$ac_includes_default" +if test "x$ac_cv_header_sys_sysctl_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_SYSCTL_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "sys/sysmacros.h" "ac_cv_header_sys_sysmacros_h" "$ac_includes_default" if test "x$ac_cv_header_sys_sysmacros_h" = xyes @@ -14020,8 +14252,8 @@ then : fi;; CYGWIN*) - LDSHARED="gcc -shared -Wl,--enable-auto-image-base" - LDCXXSHARED="g++ -shared -Wl,--enable-auto-image-base";; + LDSHARED='$(CC) -shared -Wl,--enable-auto-image-base' + LDCXXSHARED='$(CXX) -shared -Wl,--enable-auto-image-base';; *) LDSHARED="ld";; esac fi @@ -14205,17 +14437,35 @@ printf "%s\n" "$SHLIBS" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking perf trampoline" >&5 printf %s "checking perf trampoline... " >&6; } +PERF_TRAMPOLINE_OBJ="" case $PLATFORM_TRIPLET in #( x86_64-linux-gnu) : - perf_trampoline=yes ;; #( + perf_trampoline=yes + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_x86_64.o ;; #( aarch64-linux-gnu) : - perf_trampoline=yes ;; #( + perf_trampoline=yes + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_aarch64.o ;; #( darwin) : case $MACOSX_DEPLOYMENT_TARGET in #( 10.[0-9]|10.1[0-1]) : perf_trampoline=no ;; #( *) : perf_trampoline=yes + if test "${enable_universalsdk}" && test "$UNIVERSAL_ARCHS" = "universal2"; then + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_universal2.o + else + case "$host_cpu" in + x86_64) + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_x86_64.o + ;; + aarch64|arm64) + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_aarch64.o + ;; + *) + perf_trampoline=no + ;; + esac + fi ;; esac ;; #( *) : @@ -14231,7 +14481,6 @@ then : printf "%s\n" "#define PY_HAVE_PERF_TRAMPOLINE 1" >>confdefs.h - PERF_TRAMPOLINE_OBJ=Python/asm_trampoline.o fi @@ -14424,6 +14673,52 @@ fi done +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libgcc frame registration functions" >&5 +printf %s "checking for libgcc frame registration functions... " >&6; } +if test ${ac_cv_have_libgcc_eh_frame_registration+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +void __register_frame(const void *); +void __deregister_frame(const void *); + +int +main (void) +{ + +__register_frame(0); +__deregister_frame(0); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_have_libgcc_eh_frame_registration=yes +else case e in #( + e) ac_cv_have_libgcc_eh_frame_registration=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_have_libgcc_eh_frame_registration" >&5 +printf "%s\n" "$ac_cv_have_libgcc_eh_frame_registration" >&6; } +if test "x$ac_cv_have_libgcc_eh_frame_registration" = xyes +then : + + +printf "%s\n" "#define _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION 1" >>confdefs.h + + +fi + if test "x$ac_cv_require_ldl" = xyes then : @@ -16144,32 +16439,6 @@ printf "%s\n" "#define _Py_FFI_SUPPORT_C_COMPLEX 1" >>confdefs.h fi -# Check for use of the system libmpdec library -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-system-libmpdec" >&5 -printf %s "checking for --with-system-libmpdec... " >&6; } - - -# Check whether --with-system_libmpdec was given. -if test ${with_system_libmpdec+y} -then : - withval=$with_system_libmpdec; if test "x$with_system_libmpdec" = xno -then : - LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - have_mpdec=yes - with_system_libmpdec=no -fi -else case e in #( - e) with_system_libmpdec="yes" ;; -esac -fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 -printf "%s\n" "$with_system_libmpdec" >&6; } - -if test "x$with_system_libmpdec" = xyes -then : pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec >= 2.5.0" >&5 @@ -16229,26 +16498,122 @@ fi # Put the nasty error message in config.log where it belongs echo "$LIBMPDEC_PKG_ERRORS" >&5 - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} - LIBMPDEC_INTERNAL= -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} - LIBMPDEC_INTERNAL= -else - LIBMPDEC_CFLAGS=$pkg_cv_LIBMPDEC_CFLAGS - LIBMPDEC_LIBS=$pkg_cv_LIBMPDEC_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$CPPFLAGS $LIBMPDEC_CFLAGS" + LIBS="$LIBS $LIBMPDEC_LIBS" + ac_fn_c_check_header_compile "$LINENO" "mpdecimal.h" "ac_cv_header_mpdecimal_h" "$ac_includes_default" +if test "x$ac_cv_header_mpdecimal_h" = xyes +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for mpd_version in -lmpdec" >&5 +printf %s "checking for mpd_version in -lmpdec... " >&6; } +if test ${ac_cv_lib_mpdec_mpd_version+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS +LIBS="-lmpdec $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char mpd_version (void); +int +main (void) +{ +return mpd_version (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_mpdec_mpd_version=yes +else case e in #( + e) ac_cv_lib_mpdec_mpd_version=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mpdec_mpd_version" >&5 +printf "%s\n" "$ac_cv_lib_mpdec_mpd_version" >&6; } +if test "x$ac_cv_lib_mpdec_mpd_version" = xyes +then : + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include <mpdecimal.h> + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + have_mpdec=yes +else case e in #( + e) have_mpdec=no ;; +esac fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +else case e in #( + e) have_mpdec=no ;; +esac fi -if test "x$with_system_libmpdec" = xyes + +else case e in #( + e) have_mpdec=no ;; +esac +fi + + + if test "x$have_mpdec" = xyes then : + + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + save_CFLAGS=$CFLAGS save_CPPFLAGS=$CPPFLAGS save_LDFLAGS=$LDFLAGS @@ -16257,33 +16622,101 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBMPDEC_CFLAGS" LIBS="$LIBS $LIBMPDEC_LIBS" + ac_fn_c_check_header_compile "$LINENO" "mpdecimal.h" "ac_cv_header_mpdecimal_h" "$ac_includes_default" +if test "x$ac_cv_header_mpdecimal_h" = xyes +then : - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for mpd_version in -lmpdec" >&5 +printf %s "checking for mpd_version in -lmpdec... " >&6; } +if test ${ac_cv_lib_mpdec_mpd_version+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS +LIBS="-lmpdec $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char mpd_version (void); +int +main (void) +{ +return mpd_version (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_mpdec_mpd_version=yes +else case e in #( + e) ac_cv_lib_mpdec_mpd_version=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_mpdec_mpd_version" >&5 +printf "%s\n" "$ac_cv_lib_mpdec_mpd_version" >&6; } +if test "x$ac_cv_lib_mpdec_mpd_version" = xyes +then : + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - #include <mpdecimal.h> - #if MPD_VERSION_HEX < 0x02050000 - # error "mpdecimal 2.5.0 or higher required" - #endif + #include <mpdecimal.h> + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif int main (void) { -const char *x = mpd_version(); + ; return 0; } _ACEOF -if ac_fn_c_try_link "$LINENO" +if ac_fn_c_try_compile "$LINENO" then : have_mpdec=yes else case e in #( e) have_mpdec=no ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +else case e in #( + e) have_mpdec=no ;; +esac +fi + + +else case e in #( + e) have_mpdec=no ;; +esac +fi + + + if test "x$have_mpdec" = xyes +then : + + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + +fi CFLAGS=$save_CFLAGS CPPFLAGS=$save_CPPFLAGS @@ -16291,12 +16724,13 @@ LDFLAGS=$save_LDFLAGS LIBS=$save_LIBS -fi -# Disable forced inlining in debug builds, see GH-94847 -if test "x$with_pydebug" = xyes -then : - as_fn_append LIBMPDEC_CFLAGS " -DTEST_COVERAGE" +else + LIBMPDEC_CFLAGS=$pkg_cv_LIBMPDEC_CFLAGS + LIBMPDEC_LIBS=$pkg_cv_LIBMPDEC_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_mpdec=yes fi # Check whether _decimal should use a coroutine-local or thread-local context @@ -16323,90 +16757,6 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_decimal_contextvar" >&5 printf "%s\n" "$with_decimal_contextvar" >&6; } -if test "x$with_system_libmpdec" = xno -then : - # Check for libmpdec machine flavor - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for decimal libmpdec machine" >&5 -printf %s "checking for decimal libmpdec machine... " >&6; } - case $ac_sys_system in #( - Darwin*) : - libmpdec_system=Darwin ;; #( - SunOS*) : - libmpdec_system=sunos ;; #( - *) : - libmpdec_system=other - ;; -esac - - libmpdec_machine=unknown - if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal - elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi - elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - case $ac_cv_cc_name in #( - *gcc*) : - libmpdec_machine=ppro ;; #( - *clang*) : - libmpdec_machine=ppro ;; #( - *) : - libmpdec_machine=ansi32 - ;; -esac - else - libmpdec_machine=ansi32 - fi - fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libmpdec_machine" >&5 -printf "%s\n" "$libmpdec_machine" >&6; } - - case $libmpdec_machine in #( - x64) : - as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_64=1 -DASM=1" ;; #( - uint128) : - as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1" ;; #( - ansi64) : - as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_64=1 -DANSI=1" ;; #( - ppro) : - as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas" ;; #( - ansi32) : - as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_32=1 -DANSI=1" ;; #( - ansi-legacy) : - as_fn_append LIBMPDEC_CFLAGS " -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1" ;; #( - universal) : - as_fn_append LIBMPDEC_CFLAGS " -DUNIVERSAL=1" ;; #( - *) : - as_fn_error $? "_decimal: unsupported architecture" "$LINENO" 5 - ;; -esac -fi - -if test "$have_ipa_pure_const_bug" = yes; then - # Some versions of gcc miscompile inline asm: - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 - # https://gcc.gnu.org/ml/gcc/2010-11/msg00366.html - as_fn_append LIBMPDEC_CFLAGS " -fno-ipa-pure-const" -fi - -if test "$have_glibc_memmove_bug" = yes; then - # _FORTIFY_SOURCE wrappers for memmove and bcopy are incorrect: - # https://sourceware.org/ml/libc-alpha/2010-12/msg00009.html - as_fn_append LIBMPDEC_CFLAGS " -U_FORTIFY_SOURCE" -fi - - - - - @@ -19283,7 +19633,7 @@ if test "x$ac_cv_efficient_builtin_shufflevector" = xyes then : -printf "%s\n" "#define HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR 1" >>confdefs.h fi @@ -19763,12 +20113,6 @@ if test "x$ac_cv_func_dup" = xyes then : printf "%s\n" "#define HAVE_DUP 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "dup3" "ac_cv_func_dup3" -if test "x$ac_cv_func_dup3" = xyes -then : - printf "%s\n" "#define HAVE_DUP3 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "execv" "ac_cv_func_execv" if test "x$ac_cv_func_execv" = xyes @@ -20243,12 +20587,6 @@ if test "x$ac_cv_func_pipe" = xyes then : printf "%s\n" "#define HAVE_PIPE 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" -if test "x$ac_cv_func_pipe2" = xyes -then : - printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "plock" "ac_cv_func_plock" if test "x$ac_cv_func_plock" = xyes @@ -20717,6 +21055,12 @@ if test "x$ac_cv_func_sysconf" = xyes then : printf "%s\n" "#define HAVE_SYSCONF 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "sysctlbyname" "ac_cv_func_sysctlbyname" +if test "x$ac_cv_func_sysctlbyname" = xyes +then : + printf "%s\n" "#define HAVE_SYSCTLBYNAME 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "tcgetpgrp" "ac_cv_func_tcgetpgrp" if test "x$ac_cv_func_tcgetpgrp" = xyes @@ -20916,7 +21260,13 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" + ac_fn_c_check_func "$LINENO" "dup3" "ac_cv_func_dup3" +if test "x$ac_cv_func_dup3" = xyes +then : + printf "%s\n" "#define HAVE_DUP3 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" if test "x$ac_cv_func_getentropy" = xyes then : printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h @@ -20927,6 +21277,12 @@ if test "x$ac_cv_func_getgroups" = xyes then : printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" +if test "x$ac_cv_func_pipe2" = xyes +then : + printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" if test "x$ac_cv_func_system" = xyes @@ -24382,7 +24738,7 @@ printf "%s\n" "#define HAVE_DECL_PR_SET_VMA_ANON_NAME $ac_have_decl" >>confdefs. if test $ac_have_decl = 1 then : -printf "%s\n" "#define HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h +printf "%s\n" "#define _Py_HAVE_PR_SET_VMA_ANON_NAME 1" >>confdefs.h fi @@ -26673,6 +27029,27 @@ esac fi done + +ac_func= +for ac_item in $ac_func_c_list +do + if test $ac_func; then + ac_fn_c_check_func "$LINENO" $ac_func ac_cv_func_$ac_func + if eval test \"x\$ac_cv_func_$ac_func\" = xyes; then + echo "#define $ac_item 1" >> confdefs.h + fi + ac_func= + else + ac_func=$ac_item + fi +done + + + + + + + LIBS=$LIBS_SAVE { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether POSIX semaphores are enabled" >&5 @@ -29921,6 +30298,66 @@ fi + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function nofilter" >&5 +printf %s "checking for curses function nofilter... " >&6; } +if test ${ac_cv_lib_curses_nofilter+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ + + #ifndef nofilter + void *x=nofilter + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_nofilter=yes +else case e in #( + e) ac_cv_lib_curses_nofilter=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_nofilter" >&5 +printf "%s\n" "$ac_cv_lib_curses_nofilter" >&6; } + if test "x$ac_cv_lib_curses_nofilter" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_NOFILTER 1" >>confdefs.h + +fi + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_key" >&5 printf %s "checking for curses function has_key... " >&6; } if test ${ac_cv_lib_curses_has_key+y} @@ -30099,6 +30536,186 @@ fi + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function new_prescr" >&5 +printf %s "checking for curses function new_prescr... " >&6; } +if test ${ac_cv_lib_curses_new_prescr+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ + + #ifndef new_prescr + void *x=new_prescr + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_new_prescr=yes +else case e in #( + e) ac_cv_lib_curses_new_prescr=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_new_prescr" >&5 +printf "%s\n" "$ac_cv_lib_curses_new_prescr" >&6; } + if test "x$ac_cv_lib_curses_new_prescr" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_NEW_PRESCR 1" >>confdefs.h + +fi + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_screen" >&5 +printf %s "checking for curses function use_screen... " >&6; } +if test ${ac_cv_lib_curses_use_screen+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ + + #ifndef use_screen + void *x=use_screen + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_use_screen=yes +else case e in #( + e) ac_cv_lib_curses_use_screen=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_screen" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_screen" >&6; } + if test "x$ac_cv_lib_curses_use_screen" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_USE_SCREEN 1" >>confdefs.h + +fi + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_window" >&5 +printf %s "checking for curses function use_window... " >&6; } +if test ${ac_cv_lib_curses_use_window+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ + + #ifndef use_window + void *x=use_window + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_use_window=yes +else case e in #( + e) ac_cv_lib_curses_use_window=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_window" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_window" >&6; } + if test "x$ac_cv_lib_curses_use_window" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_USE_WINDOW 1" >>confdefs.h + +fi + + + CPPFLAGS=$ac_save_cppflags fi @@ -30424,7 +31041,6 @@ SRCDIRS="\ Modules \ Modules/_ctypes \ Modules/_decimal \ - Modules/_decimal/libmpdec \ Modules/_hacl \ Modules/_io \ Modules/_multiprocessing \ @@ -31523,6 +32139,50 @@ fi printf "%s\n" "$TEST_MODULES" >&6; } +# Check for --with-build-details-suffix +BUILD_DETAILS=build-details.json + +# Check whether --with-build-details-suffix was given. +if test ${with_build_details_suffix+y} +then : + withval=$with_build_details_suffix; + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-build-details-suffix" >&5 +printf %s "checking for --with-build-details-suffix... " >&6; } + if test "x$with_build_details_suffix" = xno +then : + as_fn_error $? "invalid --with-build-details-suffix option: expected custom suffix or \"yes\", not \"no\"" "$LINENO" 5 + +fi + if test "x$with_build_details_suffix" = xyes +then : + + colocated_install=yes + threading_suffix="" + if [ "$ABI_THREAD" = "t" ]; then + threading_suffix=-free-threading + fi + debug_suffix="" + if [ "$Py_DEBUG" = "true" ]; then + debug_suffix=-debug + fi + BUILD_DETAILS=build-details.$MULTIARCH$threading_suffix$debug_suffix.json + +else case e in #( + e) + BUILD_DETAILS=build-details.$with_build_details_suffix.json + + ;; +esac +fi + + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_build_details_suffix" >&5 +printf "%s\n" "$with_build_details_suffix" >&6; } +BUILD_DETAILS=$BUILD_DETAILS + + # gh-109054: Check if -latomic is needed to get <pyatomic.h> atomic functions. # On Linux aarch64, GCC may require programs and libraries to be linked # explicitly to libatomic. Call _Py_atomic_or_uint64() which may require @@ -31617,6 +32277,7 @@ case "$ac_sys_system" in iOS) _PYTHREAD_NAME_MAXLEN=63;; FreeBSD*) _PYTHREAD_NAME_MAXLEN=19;; # gh-131268 OpenBSD*) _PYTHREAD_NAME_MAXLEN=23;; # gh-131268 + CYGWIN*) _PYTHREAD_NAME_MAXLEN=16;; *) _PYTHREAD_NAME_MAXLEN=;; esac if test -n "$_PYTHREAD_NAME_MAXLEN"; then @@ -31744,6 +32405,7 @@ case $ac_sys_system in #( py_cv_module_termios=n/a py_cv_module_xxlimited=n/a py_cv_module_xxlimited_35=n/a + py_cv_module_xxlimited_3_13=n/a py_cv_module_=n/a ;; #( @@ -33727,12 +34389,7 @@ fi printf "%s\n" "$py_cv_module__decimal" >&6; } -if test "x$with_system_libmpdec" = xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library." >&5 -printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library." >&2;} -fi -if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" +if test "$have_mpdec" = "no" then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdec found; falling back to pure-Python version for the decimal module" >&5 printf "%s\n" "$as_me: WARNING: no system libmpdec found; falling back to pure-Python version for the decimal module" >&2;} @@ -34243,6 +34900,15 @@ fi printf "%s\n" "$py_cv_module__hashlib" >&6; } +case $ac_sys_system in #( + # On FreeBSD, _testcapi.get_process_memory_usage() calls kvm_openfiles() + # and so needs libkvm. + FreeBSD*) : + LIBKVM="-lkvm" + ;; #( + *) : + ;; +esac { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testcapi" >&5 printf %s "checking for stdlib extension module _testcapi... " >&6; } @@ -34269,7 +34935,7 @@ fi then : - as_fn_append MODULE_BLOCK "MODULE__TESTCAPI_LDFLAGS=$LIBATOMIC$as_nl" + as_fn_append MODULE_BLOCK "MODULE__TESTCAPI_LDFLAGS=$LIBATOMIC $LIBKVM$as_nl" fi if test "$py_cv_module__testcapi" = yes; then @@ -34708,8 +35374,8 @@ fi if test "x$py_cv_module__ctypes_test" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_CFLAGS=$LIBFFI_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBFFI_LIBS $LIBM$as_nl" + + as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBM$as_nl" fi if test "$py_cv_module__ctypes_test" = yes; then @@ -34805,6 +35471,46 @@ fi printf "%s\n" "$py_cv_module_xxlimited_35" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxlimited_3_13" >&5 +printf %s "checking for stdlib extension module xxlimited_3_13... " >&6; } + if test "$py_cv_module_xxlimited_3_13" != "n/a" +then : + + if test "$TEST_MODULES" = yes +then : + if test "$ac_cv_func_dlopen" = yes +then : + py_cv_module_xxlimited_3_13=yes +else case e in #( + e) py_cv_module_xxlimited_3_13=missing ;; +esac +fi +else case e in #( + e) py_cv_module_xxlimited_3_13=disabled ;; +esac +fi + +fi + as_fn_append MODULE_BLOCK "MODULE_XXLIMITED_3_13_STATE=$py_cv_module_xxlimited_3_13$as_nl" + if test "x$py_cv_module_xxlimited_3_13" = xyes +then : + + + + +fi + if test "$py_cv_module_xxlimited_3_13" = yes; then + MODULE_XXLIMITED_3_13_TRUE= + MODULE_XXLIMITED_3_13_FALSE='#' +else + MODULE_XXLIMITED_3_13_TRUE='#' + MODULE_XXLIMITED_3_13_FALSE= +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module_xxlimited_3_13" >&5 +printf "%s\n" "$py_cv_module_xxlimited_3_13" >&6; } + + # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" JIT_SHIM_O="" @@ -35318,6 +36024,10 @@ if test -z "${MODULE_XXLIMITED_35_TRUE}" && test -z "${MODULE_XXLIMITED_35_FALSE as_fn_error $? "conditional \"MODULE_XXLIMITED_35\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${MODULE_XXLIMITED_3_13_TRUE}" && test -z "${MODULE_XXLIMITED_3_13_FALSE}"; then + as_fn_error $? "conditional \"MODULE_XXLIMITED_3_13\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi : "${CONFIG_STATUS=./config.status}" ac_write_fail=0 @@ -35711,7 +36421,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.15, which was +This file was extended by python $as_me 3.16, which was generated by GNU Autoconf 2.72. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -35775,7 +36485,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -python config.status 3.15 +python config.status 3.16 configured by $0, generated by GNU Autoconf 2.72, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index c8cb1686d55c074..cd1883f0195c47e 100644 --- a/configure.ac +++ b/configure.ac @@ -10,7 +10,7 @@ dnl to regenerate the configure script. dnl # Set VERSION so we only need to edit in one place (i.e., here) -m4_define([PYTHON_VERSION], [3.15]) +m4_define([PYTHON_VERSION], [3.16]) AC_PREREQ([2.72]) @@ -99,7 +99,7 @@ AC_SUBST([BASECPPFLAGS]) if test "$srcdir" != . -a "$srcdir" != "$(pwd)"; then # If we're building out-of-tree, we need to make sure the following # resources get picked up before their $srcdir counterparts. - # Objects/ -> typeslots.inc + # Objects/ -> slots_generated.c # Include/ -> Python.h # (A side effect of this is that these resources will automatically be # regenerated when building out-of-tree, regardless of whether or not @@ -205,7 +205,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python], + [python$PACKAGE_VERSION python3.16 python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST([PYTHON_FOR_REGEN]) @@ -334,7 +334,7 @@ then ac_sys_system=Linux ;; *-*-cygwin*) - ac_sys_system=Cygwin + ac_sys_system=CYGWIN ;; *-apple-ios*) ac_sys_system=iOS @@ -1137,10 +1137,10 @@ preset_cxx="$CXX" if test -z "$CXX" then case "$ac_cv_cc_name" in - gcc) AC_PATH_TOOL([CXX], [g++], [g++], [notfound]) ;; - cc) AC_PATH_TOOL([CXX], [c++], [c++], [notfound]) ;; - clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; - icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; + gcc) AC_PATH_TOOL([CXX], [g++], [notfound]) ;; + cc) AC_PATH_TOOL([CXX], [c++], [notfound]) ;; + clang) AC_PATH_TOOL([CXX], [clang++], [notfound]) ;; + icc) AC_PATH_TOOL([CXX], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then @@ -1591,7 +1591,7 @@ if test $enable_shared = "yes"; then CYGWIN*) LDLIBRARY='libpython$(LDVERSION).dll.a' BLDLIBRARY='-L. -lpython$(LDVERSION)' - DLLLIBRARY='libpython$(LDVERSION).dll' + DLLLIBRARY='cygpython$(LDVERSION).dll' ;; SunOS*) LDLIBRARY='libpython$(LDVERSION).so' @@ -1650,7 +1650,7 @@ else # shared is disabled case $ac_sys_system in CYGWIN*) BLDLIBRARY='$(LIBRARY)' - LDLIBRARY='libpython$(LDVERSION).dll.a' + LDLIBRARY='libpython$(LDVERSION).a' ;; esac fi @@ -2255,7 +2255,7 @@ then -reorder-blocks=ext-tsp -reorder-functions=cdsort -split-functions - -icf=1 + -icf=0 -inline-all -split-eh -reorder-functions-use-hot-size @@ -2406,11 +2406,15 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) dnl Avoid bugs in JS fallback string decoding path AS_VAR_APPEND([LINKFORSHARED], [" -sTEXTDECODER=2"]) + dnl Workaround for a bug in Emscipten libc's getentropy. See + dnl __wrap_getentropy in Python/emscripten_syscalls.c. + AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--wrap=getentropy"]) + AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) ]) @@ -2462,6 +2466,57 @@ AS_CASE([$ac_sys_system], ] ) +dnl On Linux, check the thread stack size. musl (ex: Alpine Linux) uses +dnl a default thread stack size of 128 kB, whereas the glibc uses 8 MiB. +dnl Python needs at least 1 MiB. +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + AC_CACHE_CHECK([for thread stack size], [ac_cv_thread_stack_size], [ + cat > conftest.c <<EOF +#include <pthread.h> + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ]) + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + # Stack size used by Python/ceval.c to set Py_C_STACK_SIZE + AC_DEFINE_UNQUOTED([_Py_LINKER_THREAD_STACK_SIZE], [$ac_cv_thread_stack_size], + [Thread stack size set by the linker (in bytes).]) + fi +fi + AS_CASE([$enable_wasm_dynamic_linking], [yes], [ac_cv_func_dlopen=yes], [no], [ac_cv_func_dlopen=no], @@ -2548,9 +2603,30 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [ AX_CHECK_COMPILE_FLAG([-mno-omit-leaf-frame-pointer], [ frame_pointer_cflags="$frame_pointer_cflags -mno-omit-leaf-frame-pointer" ], [], [-Werror]) + AS_CASE([$host_cpu], [arm|armv*], [ + dnl GCC uses "-marm"; clang uses "-mno-thumb" + AX_CHECK_COMPILE_FLAG([-marm], [ + frame_pointer_cflags="$frame_pointer_cflags -marm" + ], [], [-Werror]) + AX_CHECK_COMPILE_FLAG([-mno-thumb], [ + frame_pointer_cflags="$frame_pointer_cflags -mno-thumb" + ], [], [-Werror]) + ]) + AS_CASE([$host_cpu], [powerpc64le], [ + frame_pointer_cflags="" + ]) + AS_CASE([$host_cpu], [s390*], [ + AX_CHECK_COMPILE_FLAG([-mbackchain], [ + dnl Do not use no-omit-frame-pointer; see gh-149362 + frame_pointer_cflags="-mbackchain" + ], [], [-Werror]) + ]) ], [], [-Werror]) if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then BASECFLAGS="$frame_pointer_cflags $BASECFLAGS" + AC_DEFINE([_Py_WITH_FRAME_POINTERS], [1], + [Define to 1 if frame unwinding via pointers is expected + to work, 0 if not. Leave undefined if unknown.]) fi CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" @@ -3087,7 +3163,8 @@ AC_CHECK_HEADERS([ \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/pidfd.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ - sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysmacros.h sys/termio.h sys/time.h sys/times.h sys/timerfd.h \ + sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysctl.h \ + sys/sysmacros.h sys/termio.h sys/time.h sys/times.h sys/timerfd.h \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ termios.h util.h utime.h utmp.h \ ]) @@ -3625,8 +3702,8 @@ then dnl not implemented yet ]);; CYGWIN*) - LDSHARED="gcc -shared -Wl,--enable-auto-image-base" - LDCXXSHARED="g++ -shared -Wl,--enable-auto-image-base";; + LDSHARED='$(CC) -shared -Wl,--enable-auto-image-base' + LDCXXSHARED='$(CXX) -shared -Wl,--enable-auto-image-base';; *) LDSHARED="ld";; esac fi @@ -3797,12 +3874,30 @@ AC_MSG_RESULT([$SHLIBS]) dnl perf trampoline is Linux and macOS specific and requires an arch-specific dnl trampoline in assembly. AC_MSG_CHECKING([perf trampoline]) +PERF_TRAMPOLINE_OBJ="" AS_CASE([$PLATFORM_TRIPLET], - [x86_64-linux-gnu], [perf_trampoline=yes], - [aarch64-linux-gnu], [perf_trampoline=yes], + [x86_64-linux-gnu], [perf_trampoline=yes + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_x86_64.o], + [aarch64-linux-gnu], [perf_trampoline=yes + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_aarch64.o], [darwin], [AS_CASE([$MACOSX_DEPLOYMENT_TARGET], [[10.[0-9]|10.1[0-1]]], [perf_trampoline=no], - [perf_trampoline=yes] + [perf_trampoline=yes + if test "${enable_universalsdk}" && test "$UNIVERSAL_ARCHS" = "universal2"; then + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_universal2.o + else + case "$host_cpu" in + x86_64) + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_x86_64.o + ;; + aarch64|arm64) + PERF_TRAMPOLINE_OBJ=Python/asm_trampoline_aarch64.o + ;; + *) + perf_trampoline=no + ;; + esac + fi] )], [perf_trampoline=no] ) @@ -3810,7 +3905,6 @@ AC_MSG_RESULT([$perf_trampoline]) AS_VAR_IF([perf_trampoline], [yes], [ AC_DEFINE([PY_HAVE_PERF_TRAMPOLINE], [1], [Define to 1 if you have the perf trampoline.]) - PERF_TRAMPOLINE_OBJ=Python/asm_trampoline.o ]) AC_SUBST([PERF_TRAMPOLINE_OBJ]) @@ -3828,6 +3922,24 @@ AC_CHECK_HEADERS([execinfo.h link.h dlfcn.h], [ ]) ]) +dnl for JIT GNU backtrace unwind registration +AC_CACHE_CHECK([for libgcc frame registration functions], + [ac_cv_have_libgcc_eh_frame_registration], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([[ +void __register_frame(const void *); +void __deregister_frame(const void *); +]], [[ +__register_frame(0); +__deregister_frame(0); +]])], + [ac_cv_have_libgcc_eh_frame_registration=yes], + [ac_cv_have_libgcc_eh_frame_registration=no]) + ]) +AS_VAR_IF([ac_cv_have_libgcc_eh_frame_registration], [yes], [ + AC_DEFINE([_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION], [1], + [Define to 1 if libgcc __register_frame and __deregister_frame are linkable.]) +]) + dnl only add -ldl to LDFLAGS if it isn't already part of LIBS (GH-133081) AS_VAR_IF([ac_cv_require_ldl], [yes], [ AS_VAR_IF([ac_cv_lib_dl_dlopen], [yes], [], [ @@ -4271,53 +4383,31 @@ if test "$ac_cv_ffi_complex_double_supported" = "yes"; then [Defined if _Complex C type can be used with libffi.]) fi -# Check for use of the system libmpdec library -AC_MSG_CHECKING([for --with-system-libmpdec]) -AC_DEFUN([USE_BUNDLED_LIBMPDEC], - [LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" - LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" - LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" - have_mpdec=yes - with_system_libmpdec=no]) -AC_ARG_WITH( - [system_libmpdec], - [AS_HELP_STRING( - [--with-system-libmpdec], - [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] - )], - [AS_IF([test "x$with_system_libmpdec" = xno], - [USE_BUNDLED_LIBMPDEC()])], - [with_system_libmpdec="yes"]) -AC_MSG_RESULT([$with_system_libmpdec]) - -AS_VAR_IF( - [with_system_libmpdec], [yes], - [PKG_CHECK_MODULES( - [LIBMPDEC], [libmpdec >= 2.5.0], [], - [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} - LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} - LIBMPDEC_INTERNAL=])]) - -AS_VAR_IF([with_system_libmpdec], [yes], - [WITH_SAVE_ENV([ +dnl Check for libmpdec >= 2.5.0 +PKG_CHECK_MODULES([LIBMPDEC], [libmpdec >= 2.5.0], [have_mpdec=yes], [ + WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBMPDEC_CFLAGS" LIBS="$LIBS $LIBMPDEC_LIBS" - - AC_LINK_IFELSE([ - AC_LANG_PROGRAM([ - #include <mpdecimal.h> - #if MPD_VERSION_HEX < 0x02050000 - # error "mpdecimal 2.5.0 or higher required" - #endif - ], [const char *x = mpd_version();])], - [have_mpdec=yes], - [have_mpdec=no]) - ])]) - -# Disable forced inlining in debug builds, see GH-94847 -AS_VAR_IF( - [with_pydebug], [yes], - [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DTEST_COVERAGE"])]) + AC_CHECK_HEADER([mpdecimal.h], [ + AC_CHECK_LIB([mpdec], [mpd_version], [ + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([ + #include <mpdecimal.h> + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + ], [])], + [have_mpdec=yes], + [have_mpdec=no]) + ], [have_mpdec=no]) + ], [have_mpdec=no]) + + AS_VAR_IF([have_mpdec], [yes], [ + LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} + LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} + ]) + ]) +]) # Check whether _decimal should use a coroutine-local or thread-local context AC_MSG_CHECKING([for --with-decimal-contextvar]) @@ -4338,70 +4428,6 @@ fi AC_MSG_RESULT([$with_decimal_contextvar]) -AS_VAR_IF( - [with_system_libmpdec], [no], - [# Check for libmpdec machine flavor - AC_MSG_CHECKING([for decimal libmpdec machine]) - AS_CASE([$ac_sys_system], - [Darwin*], [libmpdec_system=Darwin], - [SunOS*], [libmpdec_system=sunos], - [libmpdec_system=other] - ) - - libmpdec_machine=unknown - if test "$libmpdec_system" = Darwin; then - # universal here means: build libmpdec with the same arch options - # the python interpreter was built with - libmpdec_machine=universal - elif test $ac_cv_sizeof_size_t -eq 8; then - if test "$ac_cv_gcc_asm_for_x64" = yes; then - libmpdec_machine=x64 - elif test "$ac_cv_type___uint128_t" = yes; then - libmpdec_machine=uint128 - else - libmpdec_machine=ansi64 - fi - elif test $ac_cv_sizeof_size_t -eq 4; then - if test "$ac_cv_gcc_asm_for_x87" = yes -a "$libmpdec_system" != sunos; then - AS_CASE([$ac_cv_cc_name], - [*gcc*], [libmpdec_machine=ppro], - [*clang*], [libmpdec_machine=ppro], - [libmpdec_machine=ansi32] - ) - else - libmpdec_machine=ansi32 - fi - fi - AC_MSG_RESULT([$libmpdec_machine]) - - AS_CASE([$libmpdec_machine], - [x64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DASM=1"])], - [uint128], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1"])], - [ansi64], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_64=1 -DANSI=1"])], - [ppro], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DASM=1 -Wno-unknown-pragmas"])], - [ansi32], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1"])], - [ansi-legacy], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DCONFIG_32=1 -DANSI=1 -DLEGACY_COMPILER=1"])], - [universal], [AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -DUNIVERSAL=1"])], - [AC_MSG_ERROR([_decimal: unsupported architecture])] - )]) - -if test "$have_ipa_pure_const_bug" = yes; then - # Some versions of gcc miscompile inline asm: - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 - # https://gcc.gnu.org/ml/gcc/2010-11/msg00366.html - AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -fno-ipa-pure-const"]) -fi - -if test "$have_glibc_memmove_bug" = yes; then - # _FORTIFY_SOURCE wrappers for memmove and bcopy are incorrect: - # https://sourceware.org/ml/libc-alpha/2010-12/msg00009.html - AS_VAR_APPEND([LIBMPDEC_CFLAGS], [" -U_FORTIFY_SOURCE"]) -fi - -AC_SUBST([LIBMPDEC_CFLAGS]) -AC_SUBST([LIBMPDEC_INTERNAL]) - - dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -5124,7 +5150,7 @@ AC_LINK_IFELSE([ ]) AS_VAR_IF([ac_cv_efficient_builtin_shufflevector], [yes], [ - AC_DEFINE([HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR], [1], + AC_DEFINE([_Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR], [1], [Define if compiler supports __builtin_shufflevector with 128-bit vectors AND the target architecture has native SIMD (not just API availability)]) @@ -5382,7 +5408,7 @@ fi AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clearenv \ clock closefrom close_range confstr \ - copy_file_range ctermid dladdr dup dup3 execv explicit_bzero explicit_memset \ + copy_file_range ctermid dladdr dup execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ @@ -5392,7 +5418,7 @@ AC_CHECK_FUNCS([ \ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ - pipe2 plock poll ppoll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ + plock poll ppoll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 process_vm_readv \ pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ @@ -5406,7 +5432,7 @@ AC_CHECK_FUNCS([ \ setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ - sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ + sysconf sysctlbyname tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ tmpnam tmpnam_r truncate ttyname_r umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -5429,7 +5455,7 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - AC_CHECK_FUNCS([getentropy getgroups system]) + AC_CHECK_FUNCS([dup3 getentropy getgroups pipe2 system]) fi AC_CHECK_DECL([dirfd], @@ -5749,7 +5775,7 @@ AC_CHECK_DECLS([UT_NAMESIZE], # musl libc redefines struct prctl_mm_map and conflicts with linux/prctl.h AS_IF([test "$ac_cv_libc" != musl], [ AC_CHECK_DECLS([PR_SET_VMA_ANON_NAME], - [AC_DEFINE([HAVE_PR_SET_VMA_ANON_NAME], [1], + [AC_DEFINE([_Py_HAVE_PR_SET_VMA_ANON_NAME], [1], [Define if you have the 'PR_SET_VMA_ANON_NAME' constant.])], [], [@%:@include <linux/prctl.h> @@ -6349,6 +6375,8 @@ AC_CHECK_FUNCS( [], [AC_MSG_ERROR([Python requires C99 compatible libm])] ) + +AC_CHECK_FUNCS_ONCE(acospi asinpi atanpi atan2pi cospi sinpi tanpi) LIBS=$LIBS_SAVE dnl For multiprocessing module, check that sem_open @@ -7165,9 +7193,13 @@ PY_CHECK_CURSES_FUNC([immedok]) PY_CHECK_CURSES_FUNC([syncok]) PY_CHECK_CURSES_FUNC([wchgat]) PY_CHECK_CURSES_FUNC([filter]) +PY_CHECK_CURSES_FUNC([nofilter]) PY_CHECK_CURSES_FUNC([has_key]) PY_CHECK_CURSES_FUNC([typeahead]) PY_CHECK_CURSES_FUNC([use_env]) +PY_CHECK_CURSES_FUNC([new_prescr]) +PY_CHECK_CURSES_FUNC([use_screen]) +PY_CHECK_CURSES_FUNC([use_window]) CPPFLAGS=$ac_save_cppflags ])dnl have_curses != no ])dnl save env @@ -7355,7 +7387,6 @@ SRCDIRS="\ Modules \ Modules/_ctypes \ Modules/_decimal \ - Modules/_decimal/libmpdec \ Modules/_hacl \ Modules/_io \ Modules/_multiprocessing \ @@ -7821,6 +7852,40 @@ AC_ARG_ENABLE([test-modules], AC_MSG_RESULT([$TEST_MODULES]) AC_SUBST([TEST_MODULES]) +# Check for --with-build-details-suffix +BUILD_DETAILS=build-details.json +AC_ARG_WITH([build-details-suffix], + [AS_HELP_STRING( + [--with-build-details-suffix=], + [rename build-details.json to permit multiple colocated Python installs; optionally specify a custom suffix (default: no)] + )], + [ + AC_MSG_CHECKING([for --with-build-details-suffix]) + AS_VAR_IF( + [with_build_details_suffix], [no], + [AC_MSG_ERROR([invalid --with-build-details-suffix option: expected custom suffix or "yes", not "no"])] + ) + AS_VAR_IF( + [with_build_details_suffix], [yes], [ + colocated_install=yes + threading_suffix="" + if [[ "$ABI_THREAD" = "t" ]]; then + threading_suffix=-free-threading + fi + debug_suffix="" + if [[ "$Py_DEBUG" = "true" ]]; then + debug_suffix=-debug + fi + BUILD_DETAILS=build-details.$MULTIARCH$threading_suffix$debug_suffix.json + ], [ + BUILD_DETAILS=build-details.$with_build_details_suffix.json + ] + ) + ] +) +AC_MSG_RESULT([$with_build_details_suffix]) +AC_SUBST([BUILD_DETAILS], [$BUILD_DETAILS]) + # gh-109054: Check if -latomic is needed to get <pyatomic.h> atomic functions. # On Linux aarch64, GCC may require programs and libraries to be linked # explicitly to libatomic. Call _Py_atomic_or_uint64() which may require @@ -7896,6 +7961,7 @@ case "$ac_sys_system" in iOS) _PYTHREAD_NAME_MAXLEN=63;; FreeBSD*) _PYTHREAD_NAME_MAXLEN=19;; # gh-131268 OpenBSD*) _PYTHREAD_NAME_MAXLEN=23;; # gh-131268 + CYGWIN*) _PYTHREAD_NAME_MAXLEN=16;; *) _PYTHREAD_NAME_MAXLEN=;; esac if test -n "$_PYTHREAD_NAME_MAXLEN"; then @@ -8008,6 +8074,7 @@ AS_CASE([$ac_sys_system], [termios], [xxlimited], [xxlimited_35], + [xxlimited_3_13], ) ], [PY_STDLIB_MOD_SET_NA([_scproxy])] @@ -8329,11 +8396,7 @@ PY_STDLIB_MOD([_decimal], [], [test "$have_mpdec" = "yes"], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) -AS_VAR_IF([with_system_libmpdec], [no], - [AC_MSG_WARN([m4_normalize([ - the bundled copy of libmpdec is scheduled for removal in Python 3.16; - consider using a system installed mpdecimal library.])])]) -AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], +AS_IF([test "$have_mpdec" = "no"], [AC_MSG_WARN([m4_normalize([ no system libmpdec found; falling back to pure-Python version for the decimal module])])]) @@ -8377,10 +8440,15 @@ PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) dnl test modules +AS_CASE([$ac_sys_system], + # On FreeBSD, _testcapi.get_process_memory_usage() calls kvm_openfiles() + # and so needs libkvm. + [FreeBSD*], [LIBKVM="-lkvm"] +) PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes], dnl Modules/_testcapi needs -latomic for 32bit AIX build - [], [], [$LIBATOMIC]) + [], [], [$LIBATOMIC $LIBKVM]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic_limited], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testlimitedcapi], [test "$TEST_MODULES" = yes]) @@ -8391,14 +8459,17 @@ PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_fun PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) +dnl Check have_libffi so _ctypes_test is only built if _ctypes is built. +dnl _ctypes_test doesn't use libffi directly. PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], - [$LIBFFI_CFLAGS], [$LIBFFI_LIBS $LIBM]) + [], [$LIBM]) dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. PY_STDLIB_MOD([xxlimited], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxlimited_35], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited_3_13], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) # Determine JIT stencils header files based on target platform JIT_STENCILS_H="" diff --git a/pyconfig.h.in b/pyconfig.h.in index 9da33c954a52f8d..a05cd8ecc91e197 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -55,6 +55,9 @@ /* Define to 1 if you have the 'acosh' function. */ #undef HAVE_ACOSH +/* Define to 1 if you have the 'acospi' function. */ +#undef HAVE_ACOSPI + /* struct addrinfo (netdb.h) */ #undef HAVE_ADDRINFO @@ -73,12 +76,21 @@ /* Define to 1 if you have the 'asinh' function. */ #undef HAVE_ASINH +/* Define to 1 if you have the 'asinpi' function. */ +#undef HAVE_ASINPI + /* Define to 1 if you have the <asm/types.h> header file. */ #undef HAVE_ASM_TYPES_H +/* Define to 1 if you have the 'atan2pi' function. */ +#undef HAVE_ATAN2PI + /* Define to 1 if you have the 'atanh' function. */ #undef HAVE_ATANH +/* Define to 1 if you have the 'atanpi' function. */ +#undef HAVE_ATANPI + /* Define to 1 if you have the 'backtrace' function. */ #undef HAVE_BACKTRACE @@ -179,6 +191,9 @@ /* Define to 1 if you have the 'copy_file_range' function. */ #undef HAVE_COPY_FILE_RANGE +/* Define to 1 if you have the 'cospi' function. */ +#undef HAVE_COSPI + /* Define to 1 if you have the 'ctermid' function. */ #undef HAVE_CTERMID @@ -203,6 +218,12 @@ /* Define if you have the 'is_term_resized' function. */ #undef HAVE_CURSES_IS_TERM_RESIZED +/* Define if you have the 'new_prescr' function. */ +#undef HAVE_CURSES_NEW_PRESCR + +/* Define if you have the 'nofilter' function. */ +#undef HAVE_CURSES_NOFILTER + /* Define if you have the 'resizeterm' function. */ #undef HAVE_CURSES_RESIZETERM @@ -218,6 +239,12 @@ /* Define if you have the 'use_env' function. */ #undef HAVE_CURSES_USE_ENV +/* Define if you have the 'use_screen' function. */ +#undef HAVE_CURSES_USE_SCREEN + +/* Define if you have the 'use_window' function. */ +#undef HAVE_CURSES_USE_WINDOW + /* Define if you have the 'wchgat' function. */ #undef HAVE_CURSES_WCHGAT @@ -320,10 +347,6 @@ /* Define to 1 if you have the <editline/readline.h> header file. */ #undef HAVE_EDITLINE_READLINE_H -/* Define if compiler supports __builtin_shufflevector with 128-bit vectors - AND the target architecture has native SIMD (not just API availability) */ -#undef HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR - /* Define to 1 if you have the <endian.h> header file. */ #undef HAVE_ENDIAN_H @@ -1003,9 +1026,6 @@ /* Define if your compiler supports function prototype */ #undef HAVE_PROTOTYPES -/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ -#undef HAVE_PR_SET_VMA_ANON_NAME - /* Define to 1 if you have the 'pthread_condattr_setclock' function. */ #undef HAVE_PTHREAD_CONDATTR_SETCLOCK @@ -1262,6 +1282,9 @@ /* Define to 1 if you have the 'sigwaitinfo' function. */ #undef HAVE_SIGWAITINFO +/* Define to 1 if you have the 'sinpi' function. */ +#undef HAVE_SINPI + /* Define to 1 if you have the 'snprintf' function. */ #undef HAVE_SNPRINTF @@ -1394,6 +1417,9 @@ /* Define to 1 if you have the 'sysconf' function. */ #undef HAVE_SYSCONF +/* Define to 1 if you have the 'sysctlbyname' function. */ +#undef HAVE_SYSCTLBYNAME + /* Define to 1 if you have the <sysexits.h> header file. */ #undef HAVE_SYSEXITS_H @@ -1498,6 +1524,9 @@ /* Define to 1 if you have the <sys/syscall.h> header file. */ #undef HAVE_SYS_SYSCALL_H +/* Define to 1 if you have the <sys/sysctl.h> header file. */ +#undef HAVE_SYS_SYSCTL_H + /* Define to 1 if you have the <sys/sysmacros.h> header file. */ #undef HAVE_SYS_SYSMACROS_H @@ -1534,6 +1563,9 @@ /* Define to 1 if you have the <sys/xattr.h> header file. */ #undef HAVE_SYS_XATTR_H +/* Define to 1 if you have the 'tanpi' function. */ +#undef HAVE_TANPI + /* Define to 1 if you have the 'tcgetpgrp' function. */ #undef HAVE_TCGETPGRP @@ -2063,12 +2095,30 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define if compiler supports __builtin_shufflevector with 128-bit vectors + AND the target architecture has native SIMD (not just API availability) */ +#undef _Py_HAVE_EFFICIENT_BUILTIN_SHUFFLEVECTOR + +/* Define to 1 if libgcc __register_frame and __deregister_frame are linkable. + */ +#undef _Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION + +/* Define if you have the 'PR_SET_VMA_ANON_NAME' constant. */ +#undef _Py_HAVE_PR_SET_VMA_ANON_NAME + +/* Thread stack size set by the linker (in bytes). */ +#undef _Py_LINKER_THREAD_STACK_SIZE + /* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ #undef _Py_STACK_GROWS_DOWN /* Define if you want to use tail-calling interpreters in CPython. */ #undef _Py_TAIL_CALL_INTERP +/* Define to 1 if frame unwinding via pointers is expected to work, 0 if not. + Leave undefined if unknown. */ +#undef _Py_WITH_FRAME_POINTERS + /* Define to force use of thread-safe errno, h_errno, and other functions */ #undef _REENTRANT